standard-procedure-anvil 0.1.3.1 → 0.1.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 1768df6c04673154198ef33d8a3af8e58b330547e98816c419829f182473f1c9
-  data.tar.gz: 9a628290e842fae2e0b50c5527483904277fc9e58a1b00ca0ec4bc3f4a8e7b7b
+  metadata.gz: 50580aabd8ea329997f9a044178f08790aa12719e6d412da4a6a7a3b6c335fff
+  data.tar.gz: c4ec1dbbd6ccc1945e6fab5f76f988578bd0df0cef15cc7801b2bbdfda5f6a30
 SHA512:
-  metadata.gz: 1a166f49a00d2ae2bbd4cffa8dedc0b85bf55f50e8f5e8eb9b70a1484e3ef01ad2792d4aeafd2a04c1c84aa11f091516a59ff7ed1fde5a97e90433e204c9666e
-  data.tar.gz: b7f949bee69bc7f93497c170f20a07533cb4a22af1b19a9cc9b979c5536642294dc6ff0f18b362f03af3eebea50e1e1b13a665edcac88b98ab8a98e0c6ed4439
+  metadata.gz: e39c66044138540191d4214f5fa0b7f46d76f037b5268315f7fdb19cca214e545f93e9d89ce69d939339ca0133b7fa209281a21938ddd976d17e79160bc00167
+  data.tar.gz: 968a6df094c5f7846c23af2148b04c3671ce15c8abffce3ba407c0e3014aee4bb15e64873a6cfa0287dcd19b6193883efa259b4f8c9eb2b047abb12f6fa5ed95

data/assets/cloudinit/dokku.mysql.opensearch.ubuntu-22.yml

@@ -0,0 +1,126 @@
+#cloud-config
+users:
+  - name: %{USER}
+    groups: users, admin, docker
+    sudo: ALL=(ALL) NOPASSWD:ALL
+    shell: /bin/bash
+    ssh_authorized_keys:
+      - %{PUBLIC_KEY}
+packages:
+  - fail2ban
+  - ufw
+  - wget
+  - apt-transport-https
+  - docker.io
+  - docker-compose
+  - mysql-client
+  - libmysqlclient-dev
+package_update: true
+package_upgrade: true
+runcmd:
+  # General server setup
+  - timedatectl set-timezone UTC
+  # Prepare for OpenSearch
+  - swapoff -a
+  - echo "vm.max_map_count=262144" > /etc/sysctl.d/98-opensearch.conf
+  - sysctl -p /etc/sysctl.d/98-opensearch.conf
+  # Install MySQL
+  - echo "mysql-server mysql-server/root_password password root" | sudo debconf-set-selections
+  - echo "mysql-server mysql-server/root_password_again password root" | sudo debconf-set-selections
+  - sudo apt-get -y install mysql-server
+  - |
+    cat >> /etc/mysql/mysql.conf.d/utf8.cnf << CONF
+    [client]
+    default-character-set=utf8mb4
+
+    [mysql]
+    default-character-set=utf8mb4
+
+    [mysqld]
+    init_connect='SET collation_connection = utf8mb4_unicode_ci'
+    init_connect='SET NAMES utf8mb4'
+    character-set-server=utf8mb4
+    collation-server=utf8mb4_unicode_ci
+    skip-character-set-client-handshake
+    CONF
+  - sed -i -e '/^\(#\|\)bind-address/s/^.*$/bind-address = 0.0.0.0/' /etc/mysql/mysql.conf.d/mysqld.cnf
+  # Start MySQL
+  - systemctl start mysql.service
+  # Fail2Ban setup
+  - printf "[sshd]\nenabled = true\nbanaction = iptables-multiport" > /etc/fail2ban/jail.local
+  - systemctl enable fail2ban
+  # UFW and SSH setup
+  - ufw allow 22/tcp
+  - ufw allow 80/tcp
+  - ufw allow 443/tcp
+  - ufw enable
+  # Harden SSH
+  - sed -i -e '/^\(#\|\)PermitRootLogin/s/^.*$/PermitRootLogin no/' /etc/ssh/sshd_config
+  - sed -i -e '/^\(#\|\)PasswordAuthentication/s/^.*$/PasswordAuthentication no/' /etc/ssh/sshd_config
+  - sed -i -e '/^\(#\|\)X11Forwarding/s/^.*$/X11Forwarding no/' /etc/ssh/sshd_config
+  - sed -i -e '/^\(#\|\)MaxAuthTries/s/^.*$/MaxAuthTries 2/' /etc/ssh/sshd_config
+  - sed -i -e '/^\(#\|\)AllowTcpForwarding/s/^.*$/AllowTcpForwarding no/' /etc/ssh/sshd_config
+  - sed -i -e '/^\(#\|\)AllowAgentForwarding/s/^.*$/AllowAgentForwarding no/' /etc/ssh/sshd_config
+  - sed -i -e '/^\(#\|\)AuthorizedKeysFile/s/^.*$/AuthorizedKeysFile .ssh\/authorized_keys/' /etc/ssh/sshd_config
+  - sed -i '$a AllowUsers %{USER} dokku' /etc/ssh/sshd_config
+  # Dokku setup
+  - echo "dokku dokku/vhost_enable boolean true" | sudo debconf-set-selections
+  - wget https://dokku.com/install/v0.30.7/bootstrap.sh && sudo DOKKU_TAG=v0.30.7 bash bootstrap.sh
+  - cat /home/%{USER}/.ssh/authorized_keys | dokku ssh-keys:add admin
+  - dokku plugin:install https://github.com/dokku/dokku-cron-restart.git cron-restart
+  - dokku plugin:install https://github.com/dokku/dokku-maintenance.git maintenance
+  - dokku plugin:install https://github.com/dokku/dokku-redis.git redis
+  - dokku plugin:install https://github.com/dokku/dokku-mariadb.git mariadb
+  - dokku plugin:install https://github.com/dokku/dokku-memcached.git memcached
+  - dokku plugin:install https://github.com/dokku/dokku-letsencrypt.git letsencrypt
+  - dokku config:set --global DOKKU_LETSENCRYPT_EMAIL=sysadmin@echodek.co
+  - dokku git:set --global deploy-branch main
+  # OpenSearch setup
+  - mkdir -p /etc/opensearch
+  - docker pull opensearchproject/opensearch:latest
+  - docker pull opensearchproject/opensearch-dashboards:latest
+  - |
+    cat >> /etc/opensearch/docker-compose.yml << EOF
+    version: '3'
+    services:
+      search_db:
+        image: opensearchproject/opensearch:latest
+        container_name: search_db
+        environment:
+          - discovery.type=single-node
+          - node.name=search_db
+          - bootstrap.memory_lock=true
+          - plugins.security.disabled=true
+          - "OPENSEARCH_JAVA_OPTS=-Xms2048m -Xmx2048m"
+        ulimits:
+          memlock:
+            soft: -1
+            hard: -1
+          nofile:
+            soft: 65536
+            hard: 65536
+        volumes:
+          - opensearch_data:/usr/share/opensearch/data
+        ports:
+          - 9200:9200
+          - 9600:9600
+    volumes:
+      opensearch_data:
+    EOF
+  - |
+    cat >> /etc/systemd/system/opensearch.service << EOF
+    Description=OpenSearch container
+    Requires=docker.service
+    After=docker.service
+    [Service]
+    WorkingDirectory=/etc/opensearch
+    Restart=always
+    ExecStart=/usr/bin/docker-compose up
+    ExecStop=/usr/bin/docker-compose down
+    [Install]
+    WantedBy=multi-user.target
+    EOF
+  - systemctl daemon-reload
+  - systemctl enable opensearch.service
+  - service opensearch start
+  - reboot

data/checksums/standard-procedure-anvil-0.1.4.gem.sha512

@@ -0,0 +1 @@
+2b49b3137b8c3bbe046aa080a6bd695eedc5afe6cd6c01ff4f37a22864f76ba7c3256c5e5a4f961977eac7dac1119cc805988365a7fa505530bd0a4688cba039

data/lib/anvil/app.rb

@@ -7,7 +7,7 @@ module Anvil
   class App < Anvil::SubCommandBase
     require_relative "app/env"
 
-    desc "env", "Generate environment variables for an app"
+    desc "env /path/to/config.yml", "Generate environment variables for an app"
     long_desc <<-DESC
     List the environment variables for an app (on a given host)
 
@@ -16,13 +16,6 @@ module Anvil
 
       If the /path/to/config is not supplied, it defaults to deploy.yml
 
-      Options:
-
-      --host, -h: The server that the environment variables should be generated for - only required if multiple servers are configured
-
-      --secrets, -s: The path to a file containing secrets to be injected into the environment variables
-
-      --secrets-stdin, -S: Read secrets from STDIN instead of a file
     DESC
     option :host, type: :string, default: nil, aliases: "-h"
     option :secrets, type: :string, default: nil, aliases: "-s"
@@ -33,7 +26,7 @@ module Anvil
       puts Anvil::App::Env.new(configuration, options[:host], secrets).call
     end
 
-    desc "install", "Install an app"
+    desc "install /path/to/config.yml", "Install an app"
     long_desc <<-DESC
     Install an app on the hosts specified in the configuration.
 
@@ -45,12 +38,10 @@ module Anvil
       anvil app install /path/to/config
       If the /path/to/config is not supplied, it defaults to deploy.yml
 
-      Options:
-
-      --secrets, -s: The path to a file containing secrets to be injected into the environment variables
-
-      --secrets-stdin, -S: Read secrets from STDIN instead of a file
+      If --secrets-stdin is specified then additional environment variable values will be read from STDIN, if --secrets=/path/to/secrets is specified then they will be read from the file specified.  This is so you can specify environment variables that you do not want stored in source control.  These should be formatted as "VAR=value VAR2=value2" etc.
     DESC
+    option :secrets, type: :string, default: nil, aliases: "-s"
+    option :secrets_stdin, type: :boolean, default: false, aliases: "-S"
     def install filename = "deploy.yml"
       configuration = YAML.load_file(filename)
       secrets = read_secrets filename: options[:secrets], stdin: options[:secrets_stdin]

data/lib/anvil/cli.rb

@@ -1,14 +1,18 @@
 # frozen_string_literal: true
 
 require "thor"
-require_relative "cloudinit"
-require_relative "app"
 
 module Anvil
+  require_relative "cloudinit"
+  require_relative "app"
+  require_relative "mysql"
   class Cli < Thor
     desc "cloudinit", "Generate a cloudinit configuration"
     subcommand "cloudinit", Anvil::Cloudinit
 
+    desc "mysql", "Manage mysql"
+    subcommand "mysql", Anvil::Mysql
+
     desc "app", "Install or deploy a dokku app"
     subcommand "app", Anvil::App
 

data/lib/anvil/cloudinit.rb

@@ -13,16 +13,13 @@ module Anvil
       end
     end
 
-    desc "generate", "Generate a cloudinit configuration"
+    desc "generate configuration", "Generate a cloudinit configuration"
     long_desc <<-DESC
     Generate a cloudinit configuration for a server
 
     Example:
       anvil cloudinit generate mysql.ubuntu-22 --user dbuser --public_key ~/.ssh/my_key.pub
 
-      Options:
-      --user, -u: The user to create on the server - defaults to app
-      --public_key, -k: The path to the public key file that will be installed for the user - default to ~/.ssh/id_rsa.pub
     DESC
     option :user, type: :string, default: "app", aliases: "-u"
     option :public_key, type: :string, default: "~/.ssh/id_rsa.pub", aliases: "-k"

data/lib/anvil/mysql/create.rb

@@ -0,0 +1,70 @@
+# frozen_string_literal: true
+
+require_relative "../subcommand"
+require "rujitsu"
+module Anvil
+  class Mysql
+    require_relative "password"
+    require_relative "database_creator"
+    require_relative "user_creator"
+
+    class Create < Anvil::SubCommandBase
+      include Password
+
+      desc "database db_name user host", "Create a mysql database"
+      long_desc <<-DESC
+        Create a mysql database by SSHing into a server and then connecting to MySQL to create the database.
+
+        Example:
+
+          anvil mysql create database my_database user server.example.com --mysql-user root --mysql-host data.internal.example.com
+
+        This command will SSH into user@server.example.com, then connect to the MySQL server on data.internal.example.com as root to create the database.  It will take the root password from STDIN.
+
+        The assumption is that your MySQL server is not accessible from your local development machine.
+
+        The SSH command assumes you have a current SSH Agent to load your private key.
+
+        You can optionally supply a password for the MySQL user, or it will be read from STDIN.
+      DESC
+      option :mysql_user, type: :string, default: "root", aliases: "-m"
+      option :mysql_password, type: :string, default: nil, aliases: "-p"
+      option :mysql_host, type: :string, default: "localhost", aliases: "-H"
+      option :mysql_port, type: :numeric, default: 3306, aliases: "-P"
+      def database db_name, user, host
+        password = get_password_from options[:mysql_password]
+        Anvil::Mysql::DatabaseCreator.new(db_name, user, host, options[:mysql_user], password, options[:mysql_host], options[:mysql_port]).call
+      end
+
+      desc "user db_username user host", "Create a mysql user"
+      long_desc <<-DESC
+        Create a database user by SSHing into a server and then connecting to MySQL to create the user.
+
+        You can optionally specify a password for your database user, or it will be generated for you and returned to STDOUT.
+
+        Example:
+
+          anvil mysql create user my_user user server.example.com --mysql-user root --mysql-host data.internal.example.com
+
+        This command will SSH into user@server.example.com, then connect to the MySQL server on data.internal.example.com as root to create the user.  It will take the root password from STDIN.
+
+        The assumption is that your MySQL server is not accessible from your local development machine.
+
+        The SSH command assumes you have a current SSH Agent to load your private key.
+
+        You can optionally supply a password for the MySQL user, or it will be read from STDIN.
+      DESC
+      option :db_password, type: :string, default: nil, aliases: "-d"
+      option :mysql_user, type: :string, default: "root", aliases: "-m"
+      option :mysql_password, type: :string, default: nil, aliases: "-p"
+      option :mysql_host, type: :string, default: "localhost", aliases: "-H"
+      option :mysql_port, type: :numeric, default: 3306, aliases: "-P"
+      def user db_user, user, host
+        mysql_password = options[:mysql_password] || $stdin.gets.chomp
+        db_password = options[:db_password] || "#{4.random_letters}-#{4.random_characters}-#{4.random_numbers}-#{4.random_letters}-#{4.random_characters}"
+        Anvil::Mysql::UserCreator.new(db_user, db_password, user, host, options[:mysql_user], mysql_password, options[:mysql_host], options[:mysql_port]).call
+        puts db_password if options[:db_password].nil?
+      end
+    end
+  end
+end

data/lib/anvil/mysql/database_creator.rb

@@ -0,0 +1,25 @@
+require_relative "../ssh_executor"
+require_relative "../logger"
+require_relative "../script_runner"
+
+module Anvil
+  class Mysql
+    class DatabaseCreator < Struct.new(:db_name, :user, :host, :mysql_user, :mysql_password, :mysql_host, :mysql_port)
+      def call
+        ScriptRunner.new(script, user, host, logger).call
+      end
+
+      def db_script
+        "CREATE DATABASE #{db_name};"
+      end
+
+      def script
+        "mysql -u#{mysql_user} -p#{mysql_password} -h #{mysql_host} -P #{mysql_port} -e \"#{db_script}\""
+      end
+
+      def logger
+        Anvil::Logger.new(self.class.name)
+      end
+    end
+  end
+end

data/lib/anvil/mysql/grant.rb

@@ -0,0 +1,22 @@
+# frozen_string_literal: true
+
+require_relative "../subcommand"
+module Anvil
+  class Mysql
+    require_relative "password"
+    require_relative "privileges_granter"
+    class Grant < Anvil::SubCommandBase
+      include Password
+
+      desc "all db_name db_username user host", "Grant all privileges on db_name to db_user"
+      option :mysql_user, type: :string, default: "root", aliases: "-m"
+      option :mysql_password, type: :string, default: nil, aliases: "-p"
+      option :mysql_host, type: :string, default: "localhost", aliases: "-H"
+      option :mysql_port, type: :numeric, default: 3306, aliases: "-P"
+      def all db_name, db_username, user, host
+        password = get_password_from options[:mysql_password]
+        Anvil::Mysql::PrivilegesGranter.new(db_name, db_username, user, host, options[:mysql_user], password, options[:mysql_host], options[:mysql_port]).call
+      end
+    end
+  end
+end

data/lib/anvil/mysql/password.rb

@@ -0,0 +1,16 @@
+module Anvil
+  class Mysql
+    module Password
+      protected
+
+      def get_password_from option
+        if option.nil?
+          puts "MySQL password:"
+          $stdin.gets.chomp
+        else
+          option
+        end
+      end
+    end
+  end
+end

data/lib/anvil/mysql/privileges_granter.rb

@@ -0,0 +1,24 @@
+require_relative "../ssh_executor"
+require_relative "../logger"
+require_relative "../script_runner"
+module Anvil
+  class Mysql
+    class PrivilegesGranter < Struct.new(:db_name, :db_user, :user, :host, :mysql_user, :mysql_password, :mysql_host, :mysql_port)
+      def call
+        ScriptRunner.new(script, user, host, logger).call
+      end
+
+      def db_script
+        "GRANT ALL PRIVILEGES on #{db_name}.* to '#{db_user}'@'%';"
+      end
+
+      def script
+        "mysql -u#{mysql_user} -p#{mysql_password} -h #{mysql_host} -P #{mysql_port} -e \"#{db_script}\""
+      end
+
+      def logger
+        Anvil::Logger.new(self.class.name)
+      end
+    end
+  end
+end

data/lib/anvil/mysql/user_creator.rb

@@ -0,0 +1,24 @@
+require_relative "../ssh_executor"
+require_relative "../logger"
+require_relative "../script_runner"
+module Anvil
+  class Mysql
+    class UserCreator < Struct.new(:db_user, :db_password, :user, :host, :mysql_user, :mysql_password, :mysql_host, :mysql_port)
+      def call
+        ScriptRunner.new(script, user, host, logger).call
+      end
+
+      def db_script
+        "CREATE USER '#{db_user}'@'%' IDENTIFIED BY '#{db_password}';"
+      end
+
+      def script
+        "mysql -u#{mysql_user} -p#{mysql_password} -h #{mysql_host} -P #{mysql_port} -e \"#{db_script}\""
+      end
+
+      def logger
+        Anvil::Logger.new(self.class.name)
+      end
+    end
+  end
+end

data/lib/anvil/mysql.rb

@@ -0,0 +1,16 @@
+# frozen_string_literal: true
+
+require_relative "subcommand"
+
+module Anvil
+  class Mysql < Anvil::SubCommandBase
+    require_relative "mysql/create"
+    require_relative "mysql/grant"
+
+    desc "create", "Create mysql databases and users "
+    subcommand "create", Anvil::Mysql::Create
+
+    desc "grant", "Grant mysql permissions"
+    subcommand "grant", Anvil::Mysql::Grant
+  end
+end

data/lib/anvil/script_runner.rb

@@ -0,0 +1,12 @@
+require_relative "ssh_executor"
+module Anvil
+  class Mysql
+    class ScriptRunner < Struct.new(:script, :user, :host, :logger)
+      def call
+        SshExecutor.new(host, user, logger).call do |ssh|
+          ssh.exec! script, "SSH"
+        end
+      end
+    end
+  end
+end

data/lib/anvil/ssh_executor.rb

@@ -10,7 +10,7 @@ require "net/ssh"
 module Anvil
   class SshExecutor < Struct.new(:hostname, :user, :logger)
     def call &block
-      @connection = Net::SSH.start hostname, user, use_agent: true
+      @connection = Net::SSH.start hostname, user, use_agent: true, verify_host_key: :accept_new
       block.call self
     end
 

data/lib/anvil/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Anvil
-  VERSION = "0.1.3.1"
+  VERSION = "0.1.4"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: standard-procedure-anvil
 version: !ruby/object:Gem::Version
-  version: 0.1.3.1
+  version: 0.1.4
 platform: ruby
 authors:
 - Rahoul Baruah
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2023-06-26 00:00:00.000000000 Z
+date: 2023-06-30 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: thor
@@ -66,6 +66,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: rujitsu
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: standard-procedure-async
   requirement: !ruby/object:Gem::Requirement
@@ -96,11 +110,13 @@ files:
 - LICENSE.txt
 - README.md
 - Rakefile
+- assets/cloudinit/dokku.mysql.opensearch.ubuntu-22.yml
 - assets/cloudinit/dokku.ubuntu-22.yml
 - assets/cloudinit/memcached.ubuntu-22.yml
 - assets/cloudinit/mysql.ubuntu-22.yml
 - assets/cloudinit/opensearch.ubuntu-22.yml
 - assets/cloudinit/redis.ubuntu-22.yml
+- checksums/standard-procedure-anvil-0.1.4.gem.sha512
 - exe/anvil
 - lib/anvil.rb
 - lib/anvil/app.rb
@@ -112,6 +128,14 @@ files:
 - lib/anvil/cloudinit/generator.rb
 - lib/anvil/configuration_reader.rb
 - lib/anvil/logger.rb
+- lib/anvil/mysql.rb
+- lib/anvil/mysql/create.rb
+- lib/anvil/mysql/database_creator.rb
+- lib/anvil/mysql/grant.rb
+- lib/anvil/mysql/password.rb
+- lib/anvil/mysql/privileges_granter.rb
+- lib/anvil/mysql/user_creator.rb
+- lib/anvil/script_runner.rb
 - lib/anvil/ssh_executor.rb
 - lib/anvil/subcommand.rb
 - lib/anvil/version.rb