mrsk 0.1.0 → 0.3.0

data/lib/mrsk/cli/app.rb

@@ -23,75 +23,115 @@ class Mrsk::Cli::App < Mrsk::Cli::Base
   option :version, desc: "Defaults to the most recent git-hash in local repository"
   def start
     if (version = options[:version]).present?
-      on(MRSK.config.hosts) { execute *MRSK.app.start(version: version) }
+      on(MRSK.hosts) { execute *MRSK.app.start(version: version) }
     else
-      on(MRSK.config.hosts) { execute *MRSK.app.start, raise_on_non_zero_exit: false }
+      on(MRSK.hosts) { execute *MRSK.app.start, raise_on_non_zero_exit: false }
     end
   end
   
   desc "stop", "Stop app on servers"
   def stop
-    on(MRSK.config.hosts) { execute *MRSK.app.stop, raise_on_non_zero_exit: false }
+    on(MRSK.hosts) { execute *MRSK.app.stop, raise_on_non_zero_exit: false }
   end
   
   desc "details", "Display details about app containers"
   def details
-    on(MRSK.config.hosts) { |host| puts "App Host: #{host}\n" + capture(*MRSK.app.info, verbosity: Logger::INFO) + "\n\n" }
+    on(MRSK.hosts) { |host| puts_by_host host, capture_with_info(*MRSK.app.info) }
   end
   
-  desc "exec [CMD]", "Execute a custom task on servers passed in as CMD='bin/rake some:task'"
-  option :once, type: :boolean, default: false
+  desc "exec [CMD]", "Execute a custom command on servers"
+  option :method, aliases: "-m", default: "exec", desc: "Execution method: [exec] perform inside app container / [run] perform in new container / [ssh] perform over ssh"
   def exec(cmd)
-    if options[:once]
-      on(MRSK.config.primary_host) { puts capture(*MRSK.app.exec(cmd), verbosity: Logger::INFO) }
+    runner = \
+      case options[:method]
+      when "exec" then "exec"
+      when "run"  then "run_exec"
+      when "ssh"  then "exec_over_ssh"
+      else raise "Unknown method: #{options[:method]}"
+      end.inquiry
+
+    if runner.exec_over_ssh?
+      run_locally do
+        info "Launching command on #{MRSK.primary_host}"
+        exec MRSK.app.exec_over_ssh(cmd, host: MRSK.primary_host)
+      end
     else
-      on(MRSK.config.hosts) { |host| puts "App Host: #{host}\n" + capture(*MRSK.app.exec(cmd), verbosity: Logger::INFO) + "\n\n" }
+      on(MRSK.hosts) { |host| puts_by_host host, capture_with_info(*MRSK.app.send(runner, cmd)) }
     end
   end
-  
-  desc "console", "Start Rails Console on primary host"
-  option :host, desc: "Start console on a different host"
+
+  desc "console", "Start Rails Console on primary host (or specific host set by --hosts)"
   def console
-    host = options[:host] || MRSK.config.primary_host
+    run_locally do
+      info "Launching Rails console on #{MRSK.primary_host}"
+      exec MRSK.app.console(host: MRSK.primary_host)
+    end
+  end
 
+  desc "bash", "Start a bash session on primary host (or specific host set by --hosts)"
+  def bash
     run_locally do
-      puts "Launching Rails console on #{host}..."
-      exec MRSK.app.console(host: host)        
+      info "Launching bash session on #{MRSK.primary_host}"
+      exec MRSK.app.bash(host: MRSK.primary_host)
     end
   end
 
   desc "runner [EXPRESSION]", "Execute Rails runner with given expression"
-  option :once, type: :boolean, default: false, desc: 
   def runner(expression)
-    if options[:once]
-      on(MRSK.config.primary_host) { puts capture(*MRSK.app.exec("bin/rails", "runner", "'#{expression}'"), verbosity: Logger::INFO) }
-    else
-      on(MRSK.config.hosts) { |host| puts "App Host: #{host}\n" + capture(*MRSK.app.exec("bin/rails", "runner", "'#{expression}'"), verbosity: Logger::INFO) + "\n\n" }
-    end
+    on(MRSK.hosts) { |host| puts_by_host host, capture_with_info(*MRSK.app.exec("bin/rails", "runner", "'#{expression}'")) }
   end
 
   desc "containers", "List all the app containers currently on servers"
   def containers
-    on(MRSK.config.hosts) { |host| puts "App Host: #{host}\n" + capture(*MRSK.app.list_containers, verbosity: Logger::INFO) + "\n\n" }
+    on(MRSK.hosts) { |host| puts_by_host host, capture_with_info(*MRSK.app.list_containers) }
+  end
+
+  desc "current", "Return the current running container ID"
+  def current
+    on(MRSK.hosts) { |host| puts_by_host host, capture_with_info(*MRSK.app.current_container_id) }
   end
   
-  desc "logs", "Show last 100 log lines from app on servers"
+  desc "logs", "Show lines from app on servers"
+  option :since, aliases: "-s", desc: "Show logs since timestamp (e.g. 2013-01-02T13:23:37Z) or relative (e.g. 42m for 42 minutes)"
+  option :lines, type: :numeric, aliases: "-n", desc: "Number of log lines to pull from each server"
+  option :grep, aliases: "-g", desc: "Show lines with grep match only (use this to fetch specific requests by id)"
+  option :follow, aliases: "-f", desc: "Follow logs on primary server (or specific host set by --hosts)"
   def logs
     # FIXME: Catch when app containers aren't running
-    on(MRSK.config.hosts) { |host| puts "App Host: #{host}\n" + capture(*MRSK.app.logs) + "\n\n" }
+
+    grep = options[:grep]
+
+    if options[:follow]
+      run_locally do
+        info "Following logs on #{MRSK.primary_host}..."
+        info MRSK.app.follow_logs(host: MRSK.primary_host, grep: grep)
+        exec MRSK.app.follow_logs(host: MRSK.primary_host, grep: grep)
+      end
+    else
+      since = options[:since]
+      lines = options[:lines]
+
+      on(MRSK.hosts) do |host|
+        begin
+          puts_by_host host, capture_with_info(*MRSK.app.logs(since: since, lines: lines, grep: grep))
+        rescue SSHKit::Command::Failed
+          puts_by_host host, "Nothing found"
+        end
+      end
+    end
   end
-  
+
   desc "remove", "Remove app containers and images from servers"
   option :only, default: "", desc: "Use 'containers' or 'images'"
   def remove
     case options[:only]
     when "containers"
-      on(MRSK.config.hosts) { execute *MRSK.app.remove_containers }
+      on(MRSK.hosts) { execute *MRSK.app.remove_containers }
     when "images"
-      on(MRSK.config.hosts) { execute *MRSK.app.remove_images }
+      on(MRSK.hosts) { execute *MRSK.app.remove_images }
     else
-      on(MRSK.config.hosts) { execute *MRSK.app.remove_containers }
-      on(MRSK.config.hosts) { execute *MRSK.app.remove_images }
+      on(MRSK.hosts) { execute *MRSK.app.remove_containers }
+      on(MRSK.hosts) { execute *MRSK.app.remove_images }
     end
   end
 end

data/lib/mrsk/cli/base.rb

@@ -1,6 +1,5 @@
 require "thor"
-require "sshkit"
-require "sshkit/dsl"
+require "mrsk/sshkit_with_ext"
 
 module Mrsk::Cli
   class Base < Thor
@@ -10,12 +9,34 @@ module Mrsk::Cli
 
     class_option :verbose, type: :boolean, aliases: "-v", desc: "Detailed logging"
 
+    class_option :version, desc: "Run commands against a specific app version"
+
+    class_option :primary, type: :boolean, aliases: "-p", desc: "Run commands only on primary host instead of all"
+    class_option :hosts, aliases: "-h", desc: "Run commands on these hosts instead of all (separate by comma)"
+    class_option :roles, aliases: "-r", desc: "Run commands on these roles instead of all (separate by comma)"
+
+    class_option :config_file, aliases: "-c", default: "config/deploy.yml", desc: "Path to config file (default: config/deploy.yml)"
+    class_option :destination, aliases: "-d", desc: "Specify destination to be used for config file (west -> deploy.west.yml)"
+
     def initialize(*)
       super
-      MRSK.verbose = options[:verbose]
+      initialize_commander(options)
     end
 
     private
+      def initialize_commander(options)
+        MRSK.tap do |commander|
+          commander.config_file = Pathname.new(File.expand_path(options[:config_file]))
+          commander.destination = options[:destination]
+          commander.verbose     = options[:verbose]
+          commander.version     = options[:version]
+
+          commander.specific_hosts    = options[:hosts]&.split(",")
+          commander.specific_roles    = options[:roles]&.split(",")
+          commander.specific_primary! if options[:primary]
+        end
+      end
+
       def print_runtime
         started_at = Time.now
         yield

data/lib/mrsk/cli/build.rb

@@ -9,29 +9,45 @@ class Mrsk::Cli::Build < Mrsk::Cli::Base
 
   desc "push", "Build locally and push app image to registry"
   def push
+    verbose = options[:verbose]
+    cli = self
+
     run_locally do 
       begin
-        debug "Using builder: #{MRSK.builder.name}"
-        info "Building image may take a while (run with --verbose for progress logging)"
-        execute *MRSK.builder.push
+        MRSK.verbosity(:debug) { execute *MRSK.builder.push }
       rescue SSHKit::Command::Failed => e
-        error "Missing compatible builder, so creating a new one first"
-        execute *MRSK.builder.create
-        execute *MRSK.builder.push
+        if e.message =~ /(no builder)|(no such file or directory)/
+          error "Missing compatible builder, so creating a new one first"
+
+          if cli.create
+            MRSK.verbosity(:debug) { execute *MRSK.builder.push }
+          end
+        else
+          raise
+        end
       end
     end
   end
 
   desc "pull", "Pull app image from the registry onto servers"
   def pull
-    on(MRSK.config.hosts) { execute *MRSK.builder.pull }
+    on(MRSK.hosts) { execute *MRSK.builder.pull }
   end
 
   desc "create", "Create a local build setup"
   def create
     run_locally do
-      debug "Using builder: #{MRSK.builder.name}"
-      execute *MRSK.builder.create
+      begin
+        debug "Using builder: #{MRSK.builder.name}"
+        execute *MRSK.builder.create
+      rescue SSHKit::Command::Failed => e
+        if e.message =~ /stderr=(.*)/
+          error "Couldn't create remote builder: #{$1}"
+          false
+        else
+          raise
+        end
+      end
     end
   end
 
@@ -46,7 +62,7 @@ class Mrsk::Cli::Build < Mrsk::Cli::Base
   desc "details", "Show the name of the configured builder"
   def details
     run_locally do
-      puts "Builder: #{MRSK.builder.name} (#{MRSK.builder.target.class.name})"
+      puts "Builder: #{MRSK.builder.name}"
       puts capture(*MRSK.builder.info)
     end
   end

data/lib/mrsk/cli/main.rb

@@ -1,5 +1,6 @@
 require "mrsk/cli/base"
 
+require "mrsk/cli/accessory"
 require "mrsk/cli/app"
 require "mrsk/cli/build"
 require "mrsk/cli/prune"
@@ -8,6 +9,15 @@ require "mrsk/cli/server"
 require "mrsk/cli/traefik"
 
 class Mrsk::Cli::Main < Mrsk::Cli::Base
+  desc "setup", "Setup all accessories and deploy the app to servers"
+  def setup
+    print_runtime do
+      invoke "mrsk:cli:server:bootstrap"
+      invoke "mrsk:cli:accessory:boot", [ "all" ]
+      deploy
+    end
+  end
+
   desc "deploy", "Deploy the app to servers"
   def deploy
     print_runtime do
@@ -32,7 +42,7 @@ class Mrsk::Cli::Main < Mrsk::Cli::Base
 
   desc "rollback [VERSION]", "Rollback the app to VERSION (that must already be on servers)"
   def rollback(version)
-    on(MRSK.config.hosts) do
+    on(MRSK.hosts) do
       execute *MRSK.app.stop, raise_on_non_zero_exit: false
       execute *MRSK.app.start(version: version)
     end
@@ -42,6 +52,14 @@ class Mrsk::Cli::Main < Mrsk::Cli::Base
   def details
     invoke "mrsk:cli:traefik:details"
     invoke "mrsk:cli:app:details"
+    invoke "mrsk:cli:accessory:details", [ "all" ]
+  end
+
+  desc "config", "Show combined config"
+  def config
+    run_locally do
+      puts MRSK.config.to_h.to_yaml
+    end
   end
 
   desc "install", "Create config stub in config/deploy.yml and binstub in bin/mrsk"
@@ -79,6 +97,9 @@ class Mrsk::Cli::Main < Mrsk::Cli::Base
     puts Mrsk::VERSION
   end
 
+  desc "accessory", "Manage the accessories"
+  subcommand "accessory", Mrsk::Cli::Accessory
+
   desc "app", "Manage the application"
   subcommand "app", Mrsk::Cli::App
 

data/lib/mrsk/cli/prune.rb

@@ -9,11 +9,11 @@ class Mrsk::Cli::Prune < Mrsk::Cli::Base
 
   desc "images", "Prune unused images older than 30 days"
   def images
-    on(MRSK.config.hosts) { execute *MRSK.prune.images }
+    on(MRSK.hosts) { execute *MRSK.prune.images }
   end
 
   desc "containers", "Prune stopped containers for the service older than 3 days"
   def containers
-    on(MRSK.config.hosts) { execute *MRSK.prune.containers }
+    on(MRSK.hosts) { execute *MRSK.prune.containers }
   end
 end

data/lib/mrsk/cli/registry.rb

@@ -4,11 +4,15 @@ class Mrsk::Cli::Registry < Mrsk::Cli::Base
   desc "login", "Login to the registry locally and remotely"
   def login
     run_locally           { execute *MRSK.registry.login }
-    on(MRSK.config.hosts) { execute *MRSK.registry.login }
+    on(MRSK.hosts) { execute *MRSK.registry.login }
+  rescue ArgumentError => e
+    puts e.message
   end
 
   desc "logout", "Logout of the registry remotely"
   def logout
-    on(MRSK.config.hosts) { execute *MRSK.registry.logout }
+    on(MRSK.hosts) { execute *MRSK.registry.logout }
+  rescue ArgumentError => e
+    puts e.message
   end
 end

data/lib/mrsk/cli/server.rb

@@ -3,6 +3,6 @@ require "mrsk/cli/base"
 class Mrsk::Cli::Server < Mrsk::Cli::Base
   desc "bootstrap", "Ensure Docker is installed on the servers"
   def bootstrap
-    on(MRSK.config.hosts) { execute "which docker || apt-get install docker.io -y" }
+    on(MRSK.hosts + MRSK.accessory_hosts) { execute "which docker || (apt-get update -y && apt-get install docker.io -y)" }
   end
 end

data/lib/mrsk/cli/traefik.rb

@@ -3,17 +3,24 @@ require "mrsk/cli/base"
 class Mrsk::Cli::Traefik < Mrsk::Cli::Base
   desc "boot", "Boot Traefik on servers"
   def boot
-    on(MRSK.config.role(:web).hosts) { execute *MRSK.traefik.run, raise_on_non_zero_exit: false }
+    on(MRSK.traefik_hosts) { execute *MRSK.traefik.run, raise_on_non_zero_exit: false }
+  end
+
+  desc "reboot", "Reboot Traefik on servers (stop container, remove container, start new container)"
+  def reboot
+    invoke :stop
+    invoke :remove_container
+    invoke :boot
   end
 
   desc "start", "Start existing Traefik on servers"
   def start
-    on(MRSK.config.role(:web).hosts) { execute *MRSK.traefik.start, raise_on_non_zero_exit: false }
+    on(MRSK.traefik_hosts) { execute *MRSK.traefik.start, raise_on_non_zero_exit: false }
   end
 
   desc "stop", "Stop Traefik on servers"
   def stop
-    on(MRSK.config.role(:web).hosts) { execute *MRSK.traefik.stop, raise_on_non_zero_exit: false }
+    on(MRSK.traefik_hosts) { execute *MRSK.traefik.stop, raise_on_non_zero_exit: false }
   end
 
   desc "restart", "Restart Traefik on servers"
@@ -24,21 +31,47 @@ class Mrsk::Cli::Traefik < Mrsk::Cli::Base
 
   desc "details", "Display details about Traefik containers from servers"
   def details
-    on(MRSK.config.role(:web).hosts) { |host| puts "Traefik Host: #{host}\n" + capture(*MRSK.traefik.info, verbosity: Logger::INFO) + "\n\n" }
+    on(MRSK.traefik_hosts) { |host| puts_by_host host, capture_with_info(*MRSK.traefik.info), type: "Traefik" }
   end
 
-  desc "logs", "Show last 100 log lines from Traefik on servers"
+  desc "logs", "Show log lines from Traefik on servers"
+  option :since, aliases: "-s", desc: "Show logs since timestamp (e.g. 2013-01-02T13:23:37Z) or relative (e.g. 42m for 42 minutes)"
+  option :lines, type: :numeric, aliases: "-n", desc: "Number of log lines to pull from each server"
+  option :grep, aliases: "-g", desc: "Show lines with grep match only (use this to fetch specific requests by id)"
+  option :follow, aliases: "-f", desc: "Follow logs on primary server (or specific host set by --hosts)"
   def logs
-    on(MRSK.config.hosts) { |host| puts "Traefik Host: #{host}\n" + capture(*MRSK.traefik.logs) + "\n\n" }
+    grep = options[:grep]
+
+    if options[:follow]
+      run_locally do
+        info "Following logs on #{MRSK.primary_host}..."
+        info MRSK.traefik.follow_logs(host: MRSK.primary_host, grep: grep)
+        exec MRSK.traefik.follow_logs(host: MRSK.primary_host, grep: grep)
+      end
+    else
+      since = options[:since]
+      lines = options[:lines]
+
+      on(MRSK.traefik_hosts) do |host|
+        puts_by_host host, capture(*MRSK.traefik.logs(since: since, lines: lines, grep: grep)), type: "Traefik"
+      end
+    end
   end
 
   desc "remove", "Remove Traefik container and image from servers"
   def remove
     invoke :stop
+    invoke :remove_container
+    invoke :remove_image
+  end
 
-    on(MRSK.config.role(:web).hosts) do
-      execute *MRSK.traefik.remove_container
-      execute *MRSK.traefik.remove_image
-    end
+  desc "remove_container", "Remove Traefik container from servers"
+  def remove_container
+    on(MRSK.traefik_hosts) { execute *MRSK.traefik.remove_container }
+  end
+
+  desc "remove_container", "Remove Traefik image from servers"
+  def remove_image
+    on(MRSK.traefik_hosts) { execute *MRSK.traefik.remove_image }
   end
 end

data/lib/mrsk/cli.rb

@@ -1,9 +1,9 @@
 require "mrsk"
 
-MRSK = Mrsk::Commander.new \
-  config_file: Pathname.new(File.expand_path("config/deploy.yml"))
-
 module Mrsk::Cli
 end
 
+# SSHKit uses instance eval, so we need a global const for ergonomics
+MRSK = Mrsk::Commander.new
+
 require "mrsk/cli/main"

data/lib/mrsk/commander.rb

@@ -1,4 +1,7 @@
+require "active_support/core_ext/enumerable"
+
 require "mrsk/configuration"
+require "mrsk/commands/accessory"
 require "mrsk/commands/app"
 require "mrsk/commands/builder"
 require "mrsk/commands/prune"
@@ -6,15 +9,47 @@ require "mrsk/commands/traefik"
 require "mrsk/commands/registry"
 
 class Mrsk::Commander
-  attr_reader :config
-  attr_accessor :verbose
+  attr_accessor :config_file, :destination, :verbose, :version
 
-  def initialize(config_file:)
-    @config_file = config_file
+  def initialize(config_file: nil, destination: nil, verbose: false)
+    @config_file, @destination, @verbose = config_file, destination, verbose
   end
 
   def config
-    @config ||= Mrsk::Configuration.load_file(@config_file).tap { |config| setup_with(config) }
+    @config ||= \
+      Mrsk::Configuration
+        .create_from(config_file, destination: destination, version: cascading_version)
+        .tap { |config| configure_sshkit_with(config) }
+  end
+
+  attr_accessor :specific_hosts
+
+  def specific_primary!
+    self.specific_hosts = [ config.primary_web_host ]
+  end
+
+  def specific_roles=(role_names)
+    self.specific_hosts = config.roles.select { |r| role_names.include?(r.name) }.flat_map(&:hosts) if role_names.present?
+  end
+
+  def primary_host
+    specific_hosts&.sole || config.primary_web_host
+  end
+
+  def hosts
+    specific_hosts || config.all_hosts
+  end
+
+  def traefik_hosts
+    specific_hosts || config.traefik_hosts
+  end
+
+  def accessory_hosts
+    specific_hosts || config.accessories.collect(&:host)
+  end
+
+  def accessory_names
+    config.accessories&.collect(&:name) || []
   end
 
 
@@ -38,6 +73,10 @@ class Mrsk::Commander
     @prune ||= Mrsk::Commands::Prune.new(config)
   end
 
+  def accessory(name)
+    config.accessories.detect { |a| a.name == name }
+  end
+
 
   def verbosity(level) 
     old_level = SSHKit.config.output_verbosity
@@ -48,8 +87,12 @@ class Mrsk::Commander
   end
 
   private
+    def cascading_version
+      version.presence || ENV["VERSION"] || `git rev-parse HEAD`.strip
+    end
+
     # Lazy setup of SSHKit
-    def setup_with(config)
+    def configure_sshkit_with(config)
       SSHKit::Backend::Netssh.configure { |ssh| ssh.ssh_options = config.ssh_options }
       SSHKit.config.command_map[:docker] = "docker" # No need to use /usr/bin/env, just clogs up the logs
       SSHKit.config.output_verbosity = :debug if verbose

data/lib/mrsk/commands/accessory.rb

@@ -0,0 +1,106 @@
+require "mrsk/commands/base"
+
+class Mrsk::Commands::Accessory < Mrsk::Commands::Base
+  attr_reader :accessory_config
+  delegate :service_name, :image, :host, :port, :files, :directories, :env_args, :volume_args, :label_args, to: :accessory_config
+
+  def initialize(config, name:)
+    super(config)
+    @accessory_config = config.accessory(name)
+  end
+
+  def run
+    docker :run, 
+      "--name", service_name,
+      "-d",
+      "--restart", "unless-stopped",
+      "-p", port,
+      *env_args,
+      *volume_args,
+      *label_args,
+      image
+  end
+
+  def start
+    docker :container, :start, service_name
+  end
+
+  def stop
+    docker :container, :stop, service_name
+  end
+
+  def info
+    docker :ps, *service_filter
+  end
+
+  def logs(since: nil, lines: nil, grep: nil)
+    pipe \
+      docker(:logs, service_name, (" --since #{since}" if since), (" -n #{lines}" if lines), "-t", "2>&1"),
+      ("grep '#{grep}'" if grep)
+  end
+
+  def follow_logs(grep: nil)
+    run_over_ssh pipe(
+      docker(:logs, service_name, "-t", "-n", "10", "-f", "2>&1"),
+      ("grep '#{grep}'" if grep)
+    ).join(" "), host: host
+  end
+
+  def exec(*command, interactive: false)
+    docker :exec,
+      ("-it" if interactive),
+      *env_args,
+      *volume_args,
+      service_name,
+      *command
+  end
+
+  def run_exec(*command, interactive: false)
+    docker :run,
+      ("-it" if interactive),
+      "--rm",
+      *env_args,
+      *volume_args,
+      image,
+      *command
+  end
+
+  def bash(host:)
+    exec_over_ssh "bash", host: host
+  end
+
+  def ensure_local_file_present(local_file)
+    if !local_file.is_a?(StringIO) && !Pathname.new(local_file).exist?
+      raise "Missing file: #{local_file}"
+    end
+  end
+
+  def make_directory_for(remote_file)
+    make_directory Pathname.new(remote_file).dirname.to_s
+  end
+
+  def make_directory(path)
+    [ :mkdir, "-p", path ]
+  end
+
+  def remove_service_directory
+    [ :rm, "-rf", service_name ]
+  end
+
+  def remove_container
+    docker :container, :prune, "-f", *service_filter
+  end
+
+  def remove_image
+    docker :image, :prune, "-a", "-f", *service_filter
+  end
+
+  private
+    def exec_over_ssh(*command, host:)
+      run_over_ssh run_exec(*command, interactive: true).join(" "), host: host
+    end
+
+    def service_filter
+      [ "--filter", "label=service=#{service_name}" ]
+    end
+end