mrsk 0.0.3 → 0.2.0

data/lib/mrsk/cli/base.rb

@@ -0,0 +1,48 @@
+require "thor"
+require "mrsk/sshkit_with_ext"
+
+module Mrsk::Cli
+  class Base < Thor
+    include SSHKit::DSL
+
+    def self.exit_on_failure?() true end
+
+    class_option :verbose, type: :boolean, aliases: "-v", desc: "Detailed logging"
+
+    class_option :version, desc: "Run commands against a specific app version"
+
+    class_option :primary, type: :boolean, aliases: "-p", desc: "Run commands only on primary host instead of all"
+    class_option :hosts, aliases: "-h", desc: "Run commands on these hosts instead of all (separate by comma)"
+    class_option :roles, aliases: "-r", desc: "Run commands on these roles instead of all (separate by comma)"
+
+    class_option :config_file, aliases: "-c", default: "config/deploy.yml", desc: "Path to config file (default: config/deploy.yml)"
+    class_option :destination, aliases: "-d", desc: "Specify destination to be used for config file (west -> deploy.west.yml)"
+
+    def initialize(*)
+      super
+      initialize_commander(options)
+    end
+
+    private
+      def initialize_commander(options)
+        MRSK.tap do |commander|
+          commander.config_file = Pathname.new(File.expand_path(options[:config_file]))
+          commander.destination = options[:destination]
+          commander.verbose     = options[:verbose]
+          commander.version     = options[:version]
+
+          commander.specific_hosts    = options[:hosts]&.split(",")
+          commander.specific_roles    = options[:roles]&.split(",")
+          commander.specific_primary! if options[:primary]
+        end
+      end
+
+      def print_runtime
+        started_at = Time.now
+        yield
+      ensure
+        runtime = Time.now - started_at
+        puts "  Finished all in #{sprintf("%.1f seconds", runtime)}"
+      end
+  end
+end

data/lib/mrsk/cli/build.rb

@@ -0,0 +1,53 @@
+require "mrsk/cli/base"
+
+class Mrsk::Cli::Build < Mrsk::Cli::Base
+  desc "deliver", "Deliver a newly built app image to servers"
+  def deliver
+    invoke :push
+    invoke :pull
+  end
+
+  desc "push", "Build locally and push app image to registry"
+  def push
+    verbose = options[:verbose]
+
+    run_locally do 
+      begin
+        MRSK.verbosity(:debug) { execute *MRSK.builder.push }
+      rescue SSHKit::Command::Failed => e
+        error "Missing compatible builder, so creating a new one first"
+        execute *MRSK.builder.create
+        MRSK.verbosity(:debug) { execute *MRSK.builder.push }
+      end
+    end
+  end
+
+  desc "pull", "Pull app image from the registry onto servers"
+  def pull
+    on(MRSK.hosts) { execute *MRSK.builder.pull }
+  end
+
+  desc "create", "Create a local build setup"
+  def create
+    run_locally do
+      debug "Using builder: #{MRSK.builder.name}"
+      execute *MRSK.builder.create
+    end
+  end
+
+  desc "remove", "Remove local build setup"
+  def remove
+    run_locally do
+      debug "Using builder: #{MRSK.builder.name}"
+      execute *MRSK.builder.remove
+    end
+  end
+
+  desc "details", "Show the name of the configured builder"
+  def details
+    run_locally do
+      puts "Builder: #{MRSK.builder.name}"
+      puts capture(*MRSK.builder.info)
+    end
+  end
+end

data/lib/mrsk/cli/main.rb

@@ -0,0 +1,110 @@
+require "mrsk/cli/base"
+
+require "mrsk/cli/accessory"
+require "mrsk/cli/app"
+require "mrsk/cli/build"
+require "mrsk/cli/prune"
+require "mrsk/cli/registry"
+require "mrsk/cli/server"
+require "mrsk/cli/traefik"
+
+class Mrsk::Cli::Main < Mrsk::Cli::Base
+  desc "deploy", "Deploy the app to servers"
+  def deploy
+    print_runtime do
+      invoke "mrsk:cli:server:bootstrap"
+      invoke "mrsk:cli:registry:login"
+      invoke "mrsk:cli:build:deliver"
+      invoke "mrsk:cli:traefik:boot"
+      invoke "mrsk:cli:app:stop"
+      invoke "mrsk:cli:app:boot"
+      invoke "mrsk:cli:prune:all"
+    end
+  end
+
+  desc "redeploy", "Deploy new version of the app to servers (without bootstrapping servers, starting Traefik, pruning, and registry login)"
+  def redeploy
+    print_runtime do
+      invoke "mrsk:cli:build:deliver"
+      invoke "mrsk:cli:app:stop"
+      invoke "mrsk:cli:app:boot"
+    end
+  end
+
+  desc "rollback [VERSION]", "Rollback the app to VERSION (that must already be on servers)"
+  def rollback(version)
+    on(MRSK.hosts) do
+      execute *MRSK.app.stop, raise_on_non_zero_exit: false
+      execute *MRSK.app.start(version: version)
+    end
+  end
+
+  desc "details", "Display details about Traefik and app containers"
+  def details
+    invoke "mrsk:cli:traefik:details"
+    invoke "mrsk:cli:app:details"
+  end
+
+  desc "config", "Show combined config"
+  def config
+    run_locally do
+      pp MRSK.config.to_h
+    end
+  end
+
+  desc "install", "Create config stub in config/deploy.yml and binstub in bin/mrsk"
+  option :skip_binstub, type: :boolean, default: false, desc: "Skip adding MRSK to the Gemfile and creating bin/mrsk binstub"
+  def install
+    require "fileutils"
+
+    if (deploy_file = Pathname.new(File.expand_path("config/deploy.yml"))).exist?
+      puts "Config file already exists in config/deploy.yml (remove first to create a new one)"
+    else
+      FileUtils.cp_r Pathname.new(File.expand_path("templates/deploy.yml", __dir__)), deploy_file
+      puts "Created configuration file in config/deploy.yml"
+    end
+
+    unless options[:skip_binstub]
+      if (binstub = Pathname.new(File.expand_path("bin/mrsk"))).exist?
+        puts "Binstub already exists in bin/mrsk (remove first to create a new one)"
+      else
+        `bundle add mrsk`
+        `bundle binstubs mrsk`
+        puts "Created binstub file in bin/mrsk"
+      end
+    end
+  end
+
+  desc "remove", "Remove Traefik, app, and registry session from servers"
+  def remove
+    invoke "mrsk:cli:traefik:remove"
+    invoke "mrsk:cli:app:remove"
+    invoke "mrsk:cli:registry:logout"
+  end
+
+  desc "version", "Display the MRSK version"
+  def version
+    puts Mrsk::VERSION
+  end
+
+  desc "accessory", "Manage the accessories"
+  subcommand "accessory", Mrsk::Cli::Accessory
+
+  desc "app", "Manage the application"
+  subcommand "app", Mrsk::Cli::App
+
+  desc "build", "Build the application image"
+  subcommand "build", Mrsk::Cli::Build
+
+  desc "prune", "Prune old application images and containers"
+  subcommand "prune", Mrsk::Cli::Prune
+
+  desc "registry", "Login and out of the image registry"
+  subcommand "registry", Mrsk::Cli::Registry
+
+  desc "server", "Bootstrap servers with Docker"
+  subcommand "server", Mrsk::Cli::Server
+
+  desc "traefik", "Manage the Traefik load balancer"
+  subcommand "traefik", Mrsk::Cli::Traefik
+end

data/lib/mrsk/cli/prune.rb

@@ -0,0 +1,19 @@
+require "mrsk/cli/base"
+
+class Mrsk::Cli::Prune < Mrsk::Cli::Base
+  desc "all", "Prune unused images and stopped containers"
+  def all
+    invoke :containers
+    invoke :images
+  end
+
+  desc "images", "Prune unused images older than 30 days"
+  def images
+    on(MRSK.hosts) { execute *MRSK.prune.images }
+  end
+
+  desc "containers", "Prune stopped containers for the service older than 3 days"
+  def containers
+    on(MRSK.hosts) { execute *MRSK.prune.containers }
+  end
+end

data/lib/mrsk/cli/registry.rb

@@ -0,0 +1,18 @@
+require "mrsk/cli/base"
+
+class Mrsk::Cli::Registry < Mrsk::Cli::Base
+  desc "login", "Login to the registry locally and remotely"
+  def login
+    run_locally           { execute *MRSK.registry.login }
+    on(MRSK.hosts) { execute *MRSK.registry.login }
+  rescue ArgumentError => e
+    puts e.message
+  end
+
+  desc "logout", "Logout of the registry remotely"
+  def logout
+    on(MRSK.hosts) { execute *MRSK.registry.logout }
+  rescue ArgumentError => e
+    puts e.message
+  end
+end

data/lib/mrsk/cli/server.rb

@@ -0,0 +1,8 @@
+require "mrsk/cli/base"
+
+class Mrsk::Cli::Server < Mrsk::Cli::Base
+  desc "bootstrap", "Ensure Docker is installed on the servers"
+  def bootstrap
+    on(MRSK.hosts) { execute "which docker || (apt-get update -y && apt-get install docker.io -y)" }
+  end
+end

data/lib/mrsk/cli/templates/deploy.yml

@@ -0,0 +1,17 @@
+# Name of your application. Used to uniquely configuring Traefik and app containers.
+# Your Dockerfile should set LABEL service=the-same-value to ensure image pruning works.
+service: my-app
+
+# Name of the container image.
+image: user/my-app
+
+# Deploy to these servers.
+servers:
+  - 192.168.0.1
+
+# Credentials for your image host.
+registry:
+  # Specify the registry server, if you're not using Docker Hub
+  # server: registry.digitalocean.com / ghcr.io / ...
+  username: my-user
+  password: my-password-should-go-somewhere-safe

data/lib/mrsk/cli/traefik.rb

@@ -0,0 +1,77 @@
+require "mrsk/cli/base"
+
+class Mrsk::Cli::Traefik < Mrsk::Cli::Base
+  desc "boot", "Boot Traefik on servers"
+  def boot
+    on(MRSK.traefik_hosts) { execute *MRSK.traefik.run, raise_on_non_zero_exit: false }
+  end
+
+  desc "reboot", "Reboot Traefik on servers (stop container, remove container, start new container)"
+  def reboot
+    invoke :stop
+    invoke :remove_container
+    invoke :boot
+  end
+
+  desc "start", "Start existing Traefik on servers"
+  def start
+    on(MRSK.traefik_hosts) { execute *MRSK.traefik.start, raise_on_non_zero_exit: false }
+  end
+
+  desc "stop", "Stop Traefik on servers"
+  def stop
+    on(MRSK.traefik_hosts) { execute *MRSK.traefik.stop, raise_on_non_zero_exit: false }
+  end
+
+  desc "restart", "Restart Traefik on servers"
+  def restart
+    invoke :stop
+    invoke :start
+  end
+
+  desc "details", "Display details about Traefik containers from servers"
+  def details
+    on(MRSK.traefik_hosts) { |host| puts_by_host host, capture_with_info(*MRSK.traefik.info), type: "Traefik" }
+  end
+
+  desc "logs", "Show log lines from Traefik on servers"
+  option :since, aliases: "-s", desc: "Show logs since timestamp (e.g. 2013-01-02T13:23:37Z) or relative (e.g. 42m for 42 minutes)"
+  option :lines, type: :numeric, aliases: "-n", desc: "Number of log lines to pull from each server"
+  option :grep, aliases: "-g", desc: "Show lines with grep match only (use this to fetch specific requests by id)"
+  option :follow, aliases: "-f", desc: "Follow logs on primary server (or specific host set by --hosts)"
+  def logs
+    grep = options[:grep]
+
+    if options[:follow]
+      run_locally do
+        info "Following logs on #{MRSK.primary_host}..."
+        info MRSK.traefik.follow_logs(host: MRSK.primary_host, grep: grep)
+        exec MRSK.traefik.follow_logs(host: MRSK.primary_host, grep: grep)
+      end
+    else
+      since = options[:since]
+      lines = options[:lines]
+
+      on(MRSK.traefik_hosts) do |host|
+        puts_by_host host, capture(*MRSK.traefik.logs(since: since, lines: lines, grep: grep)), type: "Traefik"
+      end
+    end
+  end
+
+  desc "remove", "Remove Traefik container and image from servers"
+  def remove
+    invoke :stop
+    invoke :remove_container
+    invoke :remove_image
+  end
+
+  desc "remove_container", "Remove Traefik container from servers"
+  def remove_container
+    on(MRSK.traefik_hosts) { execute *MRSK.traefik.remove_container }
+  end
+
+  desc "remove_container", "Remove Traefik image from servers"
+  def remove_image
+    on(MRSK.traefik_hosts) { execute *MRSK.traefik.remove_image }
+  end
+end

data/lib/mrsk/cli.rb

@@ -0,0 +1,9 @@
+require "mrsk"
+
+module Mrsk::Cli
+end
+
+# SSHKit uses instance eval, so we need a global const for ergonomics
+MRSK = Mrsk::Commander.new
+
+require "mrsk/cli/main"

data/lib/mrsk/commander.rb

@@ -1,4 +1,7 @@
+require "active_support/core_ext/enumerable"
+
 require "mrsk/configuration"
+require "mrsk/commands/accessory"
 require "mrsk/commands/app"
 require "mrsk/commands/builder"
 require "mrsk/commands/prune"
@@ -6,14 +9,43 @@ require "mrsk/commands/traefik"
 require "mrsk/commands/registry"
 
 class Mrsk::Commander
-  attr_reader :config_file, :config, :verbose
+  attr_accessor :config_file, :destination, :verbose, :version
 
-  def initialize(config_file:, verbose: false)
-    @config_file, @verbose = config_file, verbose
+  def initialize(config_file: nil, destination: nil, verbose: false)
+    @config_file, @destination, @verbose = config_file, destination, verbose
   end
 
   def config
-    @config ||= Mrsk::Configuration.load_file(config_file).tap { |config| setup_with(config) }
+    @config ||= \
+      Mrsk::Configuration
+        .create_from(config_file, destination: destination, version: cascading_version)
+        .tap { |config| configure_sshkit_with(config) }
+  end
+
+  attr_accessor :specific_hosts
+
+  def specific_primary!
+    self.specific_hosts = [ config.primary_web_host ]
+  end
+
+  def specific_roles=(role_names)
+    self.specific_hosts = config.roles.select { |r| role_names.include?(r.name) }.flat_map(&:hosts) if role_names.present?
+  end
+
+  def primary_host
+    specific_hosts&.sole || config.primary_web_host
+  end
+
+  def hosts
+    specific_hosts || config.all_hosts
+  end
+
+  def traefik_hosts
+    specific_hosts || config.traefik_hosts
+  end
+
+  def accessory_hosts
+    specific_hosts || config.accessories.collect(&:host)
   end
 
 
@@ -37,6 +69,10 @@ class Mrsk::Commander
     @prune ||= Mrsk::Commands::Prune.new(config)
   end
 
+  def accessory(name)
+    (@accessories ||= {})[name] ||= Mrsk::Commands::Accessory.new(config, name: name)
+  end
+
 
   def verbosity(level) 
     old_level = SSHKit.config.output_verbosity
@@ -47,8 +83,12 @@ class Mrsk::Commander
   end
 
   private
+    def cascading_version
+      version.presence || ENV["VERSION"] || `git rev-parse HEAD`.strip
+    end
+
     # Lazy setup of SSHKit
-    def setup_with(config)
+    def configure_sshkit_with(config)
       SSHKit::Backend::Netssh.configure { |ssh| ssh.ssh_options = config.ssh_options }
       SSHKit.config.command_map[:docker] = "docker" # No need to use /usr/bin/env, just clogs up the logs
       SSHKit.config.output_verbosity = :debug if verbose

data/lib/mrsk/commands/accessory.rb

@@ -0,0 +1,61 @@
+require "mrsk/commands/base"
+
+class Mrsk::Commands::Accessory < Mrsk::Commands::Base
+  attr_reader :accessory_config
+  delegate :service_name, :image, :host, :port, :env_args, :volume_args, :label_args, to: :accessory_config
+
+  def initialize(config, name:)
+    super(config)
+    @accessory_config = config.accessory(name)
+  end
+
+  def run
+    docker :run, 
+      "--name", service_name,
+      "-d",
+      "--restart", "unless-stopped",
+      "-p", port,
+      *env_args,
+      *volume_args,
+      *label_args,
+      image
+  end
+
+  def start
+    docker :container, :start, service_name
+  end
+
+  def stop
+    docker :container, :stop, service_name
+  end
+
+  def info
+    docker :ps, *service_filter
+  end
+
+  def logs(since: nil, lines: nil, grep: nil)
+    pipe \
+      docker(:logs, service_name, (" --since #{since}" if since), (" -n #{lines}" if lines), "-t", "2>&1"),
+      ("grep '#{grep}'" if grep)
+  end
+
+  def follow_logs(grep: nil)
+    run_over_ssh pipe(
+      docker(:logs, service_name, "-t", "-n", "10", "-f", "2>&1"),
+      ("grep '#{grep}'" if grep)
+    ).join(" "), host: host
+  end
+
+  def remove_container
+    docker :container, :prune, "-f", *service_filter
+  end
+
+  def remove_image
+    docker :image, :prune, "-a", "-f", *service_filter
+  end
+
+  private
+    def service_filter
+      [ "--filter", "label=service=#{service_name}" ]
+    end
+end

data/lib/mrsk/commands/app.rb

@@ -8,40 +8,72 @@ class Mrsk::Commands::App < Mrsk::Commands::Base
       "-d",
       "--restart unless-stopped",
       "--name", config.service_with_version,
-      "-e", redact("RAILS_MASTER_KEY=#{config.master_key}"),
-      *config.env_args,
+      *rails_master_key_arg,
+      *role.env_args,
+      *config.volume_args,
       *role.label_args,
       config.absolute_image,
       role.cmd
   end
 
-  def start
-    docker :start, config.service_with_version
+  def start(version: config.version)
+    docker :start, "#{config.service}-#{version}"
+  end
+
+  def current_container_id
+    docker :ps, "-q", *service_filter
   end
 
   def stop
-    [ "docker ps -q #{service_filter.join(" ")} | xargs docker stop" ]
+    pipe current_container_id, "xargs docker stop"
   end
 
   def info
     docker :ps, *service_filter
   end
 
-  def logs
-    [ "docker ps -q #{service_filter.join(" ")} | xargs docker logs -n 100 -t" ]
+  def logs(since: nil, lines: nil, grep: nil)
+    pipe \
+      current_container_id,
+      "xargs docker logs#{" --since #{since}" if since}#{" -n #{lines}" if lines} -t 2>&1",
+      ("grep '#{grep}'" if grep)
   end
 
   def exec(*command, interactive: false)
     docker :exec,
       ("-it" if interactive),
-      "-e", redact("RAILS_MASTER_KEY=#{config.master_key}"),
+      *rails_master_key_arg,
       *config.env_args,
+      *config.volume_args,
       config.service_with_version,
       *command
   end
 
-  def console
-    "ssh -t #{config.ssh_user}@#{config.primary_host} '#{exec("bin/rails", "c", interactive: true).join(" ")}'"
+  def run_exec(*command, interactive: false)
+    docker :run,
+      ("-it" if interactive),
+      "--rm",
+      *rails_master_key_arg,
+      *config.env_args,
+      *config.volume_args,
+      config.absolute_image,
+      *command
+  end
+
+  def follow_logs(host:, grep: nil)
+    run_over_ssh pipe(
+      current_container_id,
+      "xargs docker logs -t -n 10 -f 2>&1",
+      ("grep '#{grep}'" if grep)
+    ).join(" "), host: host
+  end
+
+  def console(host:)
+    exec_over_ssh "bin/rails", "c", host: host
+  end
+
+  def bash(host:)
+    exec_over_ssh "bash", host: host
   end
 
   def list_containers
@@ -57,7 +89,15 @@ class Mrsk::Commands::App < Mrsk::Commands::Base
   end
 
   private
+    def exec_over_ssh(*command, host:)
+      run_over_ssh run_exec(*command, interactive: true).join(" "), host: host
+    end
+
     def service_filter
       [ "--filter", "label=service=#{config.service}" ]
     end
+
+    def rails_master_key_arg
+      [ "-e", redact("RAILS_MASTER_KEY=#{config.master_key}") ]
+    end
 end

data/lib/mrsk/commands/base.rb

@@ -1,7 +1,7 @@
-require "sshkit"
-
 module Mrsk::Commands
   class Base
+    delegate :redact, to: Mrsk::Utils
+
     attr_accessor :config
 
     def initialize(config)
@@ -9,19 +9,23 @@ module Mrsk::Commands
     end
 
     private
-      def combine(*commands)
+      def combine(*commands, by: "&&")
         commands
-          .collect { |command| command + [ "&&" ] }.flatten # Join commands with &&
-          .tap     { |commands| commands.pop } # Remove trailing &&
+          .compact
+          .collect { |command| Array(command) + [ by ] }.flatten # Join commands
+          .tap     { |commands| commands.pop } # Remove trailing combiner
+      end
+
+      def pipe(*commands)
+        combine *commands, by: "|"
       end
 
       def docker(*args)
         args.compact.unshift :docker
       end
 
-      # Copied from SSHKit::Backend::Abstract#redact to be available inside Commands classes
-      def redact(arg) # Used in execute_command to hide redact() args a user passes in
-        arg.to_s.extend(SSHKit::Redaction) # to_s due to our inability to extend Integer, etc
-      end      
+      def run_over_ssh(command, host:)
+        "ssh -t #{config.ssh_user}@#{host} '#{command}'"
+      end
   end
 end

data/lib/mrsk/commands/builder/base.rb

@@ -0,0 +1,26 @@
+require "mrsk/commands/base"
+
+class Mrsk::Commands::Builder::Base < Mrsk::Commands::Base
+  delegate :argumentize, to: Mrsk::Utils
+
+  def pull
+    docker :pull, config.absolute_image
+  end
+
+  def build_args
+    argumentize "--build-arg", args, redacted: true
+  end
+
+  def build_secrets
+    argumentize "--secret", secrets.collect { |secret| [ "id", secret ] }
+  end
+
+  private
+    def args
+      (config.builder && config.builder["args"]) || {}
+    end
+
+    def secrets
+      (config.builder && config.builder["secrets"]) || []
+    end
+end