mrsk 0.2.0 → 0.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 04a17f68f19cfaf130747feded6450c389fdcb01bb83c4d44b1acb0c33d6fb60
-  data.tar.gz: 4d60e6c22acd3bf07fa22b0e488138166cf2265cedcd102ebb3245bb6318f5e0
+  metadata.gz: a3cbef66064bc4d1a20707788eefd3827d4054d1278d85a8e2b425c4c5ad643e
+  data.tar.gz: 26e48f594d93f1b1a409a4601cc43c6d62fc35e935c7000c17addd6d5f720ac9
 SHA512:
-  metadata.gz: 379a355dafd696bbe470367c712e318e6ff9b215143059bde147dc1fdca2870bc7e46a1f47057976c2c6a5ec2601ca7cef722d928c33d3b0f1ee0700a1ee934c
-  data.tar.gz: b0a6a7e0efa5824e116c3728726a3be1d6c9310903eefa992411a2d91731dddd8db6903d6a8697c5d5643b831b39adad9ec088ff28262d5b79e4f4aad3a90c46
+  metadata.gz: 3c84c6bada6dc82715349630807685d4f8a6c5dd64995760efd88666bc99a1c1b20230f47b23a7b71f8dfab5f4b2f34317d1fd2a5245b8c8a6b45575fa606002
+  data.tar.gz: 83565d28bcde4d782a829312685c2669512f6111233685309b015ed2b087ac5557456faae0460f472b18c480ef20892ae712a07f1b22de2cb2340a9f187c936b

data/README.md

@@ -244,6 +244,14 @@ ARG RUBY_VERSION
 FROM ruby:$RUBY_VERSION-slim as base
 ```
 
+### Using without RAILS_MASTER_KEY
+
+If you're using MRSK with older Rails apps that predate RAILS_MASTER_KEY, or with a non-Rails app, you can skip the default usage and reference:
+
+```yaml
+skip_master_key: true
+```
+
 ### Using accessories for database, cache, search services
 
 You can manage your accessory services via MRSK as well. The services will build off public images, and will not be automatically updated when you deploy:

data/bin/mrsk

@@ -1,5 +1,13 @@
 #!/usr/bin/env ruby
 
+# Prevent failures from being reported twice.
+Thread.report_on_exception = false
+
 require "mrsk/cli"
 
-Mrsk::Cli::Main.start(ARGV)
+begin
+  Mrsk::Cli::Main.start(ARGV)
+rescue SSHKit::Runner::ExecuteError => e
+  puts "  \e[31mERROR (#{e.cause.class}): #{e.cause.message}\e[0m"
+  puts e.cause.backtrace if ENV["VERBOSE"]
+end

data/lib/mrsk/cli/accessory.rb

@@ -1,41 +1,104 @@
 require "mrsk/cli/base"
 
 class Mrsk::Cli::Accessory < Mrsk::Cli::Base
-  desc "boot [NAME]", "Boot accessory service on host"
+  desc "boot [NAME]", "Boot accessory service on host (use NAME=all to boot all accessories)"
   def boot(name)
-    accessory = MRSK.accessory(name)
-    on(accessory.host) { execute *accessory.run }
+    if name == "all"
+      MRSK.accessory_names.each { |accessory_name| boot(accessory_name) }
+    else
+      with_accessory(name) do |accessory|
+        directories(name)
+        upload(name)
+        on(accessory.host) { execute *accessory.run }
+      end
+    end
+  end
+
+  desc "upload [NAME]", "Upload accessory files to host"
+  def upload(name)
+    with_accessory(name) do |accessory|
+      on(accessory.host) do
+        accessory.files.each do |(local, remote)|
+          accessory.ensure_local_file_present(local)
+
+          execute *accessory.make_directory_for(remote)
+          upload! local, remote
+          execute :chmod, "755", remote
+        end
+      end
+    end
+  end
+
+  desc "directories [NAME]", "Create accessory directories on host"
+  def directories(name)
+    with_accessory(name) do |accessory|
+      on(accessory.host) do
+        accessory.directories.keys.each do |host_path|
+          execute *accessory.make_directory(host_path)
+        end
+      end
+    end
   end
 
   desc "reboot [NAME]", "Reboot accessory on host (stop container, remove container, start new container)"
   def reboot(name)
-    invoke :stop, [ name ]
-    invoke :remove_container, [ name ]
-    invoke :boot, [ name ]
+    with_accessory(name) do |accessory|
+      stop(name)
+      remove_container(name)
+      boot(name)
+    end
   end
 
   desc "start [NAME]", "Start existing accessory on host"
   def start(name)
-    accessory = MRSK.accessory(name)
-    on(accessory.host) { execute *accessory.start }
+    with_accessory(name) do |accessory|
+      on(accessory.host) { execute *accessory.start }
+    end
   end
 
   desc "stop [NAME]", "Stop accessory on host"
   def stop(name)
-    accessory = MRSK.accessory(name)
-    on(accessory.host) { execute *accessory.stop }
+    with_accessory(name) do |accessory|
+      on(accessory.host) { execute *accessory.stop, raise_on_non_zero_exit: false }
+    end
   end
 
   desc "restart [NAME]", "Restart accessory on host"
   def restart(name)
-    invoke :stop, [ name ]
-    invoke :start, [ name ]
+    with_accessory(name) do
+      stop(name)
+      start(name)
+    end
   end
 
-  desc "details [NAME]", "Display details about accessory on host"
+  desc "details [NAME]", "Display details about accessory on host (use NAME=all to boot all accessories)"
   def details(name)
-    accessory = MRSK.accessory(name)
-    on(accessory.host) { puts capture_with_info(*accessory.info) }
+    if name == "all"
+      MRSK.accessory_names.each { |accessory_name| details(accessory_name) }
+    else
+      with_accessory(name) do |accessory|
+        on(accessory.host) { puts capture_with_info(*accessory.info) }
+      end
+    end
+  end
+
+  desc "exec [NAME] [CMD]", "Execute a custom command on accessory host"
+  option :run, type: :boolean, default: false, desc: "Start a new container to run the command rather than reusing existing"
+  def exec(name, cmd)
+    with_accessory(name) do |accessory|
+      runner = options[:run] ? :run_exec : :exec
+      on(accessory.host) { |host| puts_by_host host, capture_with_info(*accessory.send(runner, cmd)) }
+    end
+  end
+
+  desc "bash [NAME]", "Start a bash session on primary host (or specific host set by --hosts)"
+  def bash(name)
+    with_accessory(name) do |accessory|
+      run_locally do
+        info "Launching bash session on #{accessory.host}"
+        exec accessory.bash(host: accessory.host)
+      end
+    end
   end
 
   desc "logs [NAME]", "Show log lines from accessory on host"
@@ -44,42 +107,75 @@ class Mrsk::Cli::Accessory < Mrsk::Cli::Base
   option :grep, aliases: "-g", desc: "Show lines with grep match only (use this to fetch specific requests by id)"
   option :follow, aliases: "-f", desc: "Follow logs on primary server (or specific host set by --hosts)"
   def logs(name)
-    accessory = MRSK.accessory(name)
-
-    grep = options[:grep]
+    with_accessory(name) do |accessory|
+      grep = options[:grep]
 
-    if options[:follow]
-      run_locally do
-        info "Following logs on #{accessory.host}..."
-        info accessory.follow_logs(grep: grep)
-        exec accessory.follow_logs(grep: grep)
-      end
-    else
-      since = options[:since]
-      lines = options[:lines]
+      if options[:follow]
+        run_locally do
+          info "Following logs on #{accessory.host}..."
+          info accessory.follow_logs(grep: grep)
+          exec accessory.follow_logs(grep: grep)
+        end
+      else
+        since = options[:since]
+        lines = options[:lines]
 
-      on(accessory.host) do
-        puts capture_with_info(*accessory.logs(since: since, lines: lines, grep: grep))
+        on(accessory.host) do
+          puts capture_with_info(*accessory.logs(since: since, lines: lines, grep: grep))
+        end
       end
     end
   end
 
-  desc "remove [NAME]", "Remove accessory container and image from host"
+  desc "remove [NAME]", "Remove accessory container and image from host (use NAME=all to boot all accessories)"
   def remove(name)
-    invoke :stop, [ name ]
-    invoke :remove_container, [ name ]
-    invoke :remove_image, [ name ]
+    if name == "all"
+      MRSK.accessory_names.each { |accessory_name| remove(accessory_name) }
+    else
+      with_accessory(name) do
+        stop(name)
+        remove_container(name)
+        remove_image(name)
+        remove_service_directory(name)
+      end
+    end
   end
 
   desc "remove_container [NAME]", "Remove accessory container from host"
   def remove_container(name)
-    accessory = MRSK.accessory(name)
-    on(accessory.host) { execute *accessory.remove_container }
+    with_accessory(name) do |accessory|
+      on(accessory.host) { execute *accessory.remove_container }
+    end
   end
 
-  desc "remove_container [NAME]", "Remove accessory image from servers"
+  desc "remove_container [NAME]", "Remove accessory image from host"
   def remove_image(name)
-    accessory = MRSK.accessory(name)
-    on(accessory.host) { execute *accessory.remove_image }
+    with_accessory(name) do |accessory|
+      on(accessory.host) { execute *accessory.remove_image }
+    end
+  end
+
+  desc "remove_service_directory [NAME]", "Remove accessory directory used for uploaded files and data directories from host"
+  def remove_service_directory(name)
+    with_accessory(name) do |accessory|
+      on(accessory.host) { execute *accessory.remove_service_directory }
+    end
   end
+
+  private
+    def with_accessory(name)
+      if accessory = MRSK.accessory(name)
+        yield accessory
+      else
+        error_on_missing_accessory(name)
+      end
+    end
+
+    def error_on_missing_accessory(name)
+      options = MRSK.accessory_names.presence
+
+      error \
+        "No accessory by the name of '#{name}'" +
+        (options ? " (options: #{options.to_sentence})" : "")
+    end
 end

data/lib/mrsk/cli/app.rb

@@ -40,10 +40,24 @@ class Mrsk::Cli::App < Mrsk::Cli::Base
   end
   
   desc "exec [CMD]", "Execute a custom command on servers"
-  option :run, type: :boolean, default: false, desc: "Start a new container to run the command rather than reusing existing"
+  option :method, aliases: "-m", default: "exec", desc: "Execution method: [exec] perform inside app container / [run] perform in new container / [ssh] perform over ssh"
   def exec(cmd)
-    runner = options[:run] ? :run_exec : :exec
-    on(MRSK.hosts) { |host| puts_by_host host, capture_with_info(*MRSK.app.send(runner, cmd)) }
+    runner = \
+      case options[:method]
+      when "exec" then "exec"
+      when "run"  then "run_exec"
+      when "ssh"  then "exec_over_ssh"
+      else raise "Unknown method: #{options[:method]}"
+      end.inquiry
+
+    if runner.exec_over_ssh?
+      run_locally do
+        info "Launching command on #{MRSK.primary_host}"
+        exec MRSK.app.exec_over_ssh(cmd, host: MRSK.primary_host)
+      end
+    else
+      on(MRSK.hosts) { |host| puts_by_host host, capture_with_info(*MRSK.app.send(runner, cmd)) }
+    end
   end
 
   desc "console", "Start Rails Console on primary host (or specific host set by --hosts)"

data/lib/mrsk/cli/build.rb

@@ -10,14 +10,21 @@ class Mrsk::Cli::Build < Mrsk::Cli::Base
   desc "push", "Build locally and push app image to registry"
   def push
     verbose = options[:verbose]
+    cli = self
 
     run_locally do 
       begin
         MRSK.verbosity(:debug) { execute *MRSK.builder.push }
       rescue SSHKit::Command::Failed => e
-        error "Missing compatible builder, so creating a new one first"
-        execute *MRSK.builder.create
-        MRSK.verbosity(:debug) { execute *MRSK.builder.push }
+        if e.message =~ /(no builder)|(no such file or directory)/
+          error "Missing compatible builder, so creating a new one first"
+
+          if cli.create
+            MRSK.verbosity(:debug) { execute *MRSK.builder.push }
+          end
+        else
+          raise
+        end
       end
     end
   end
@@ -30,8 +37,17 @@ class Mrsk::Cli::Build < Mrsk::Cli::Base
   desc "create", "Create a local build setup"
   def create
     run_locally do
-      debug "Using builder: #{MRSK.builder.name}"
-      execute *MRSK.builder.create
+      begin
+        debug "Using builder: #{MRSK.builder.name}"
+        execute *MRSK.builder.create
+      rescue SSHKit::Command::Failed => e
+        if e.message =~ /stderr=(.*)/
+          error "Couldn't create remote builder: #{$1}"
+          false
+        else
+          raise
+        end
+      end
     end
   end
 

data/lib/mrsk/cli/main.rb

@@ -9,6 +9,15 @@ require "mrsk/cli/server"
 require "mrsk/cli/traefik"
 
 class Mrsk::Cli::Main < Mrsk::Cli::Base
+  desc "setup", "Setup all accessories and deploy the app to servers"
+  def setup
+    print_runtime do
+      invoke "mrsk:cli:server:bootstrap"
+      invoke "mrsk:cli:accessory:boot", [ "all" ]
+      deploy
+    end
+  end
+
   desc "deploy", "Deploy the app to servers"
   def deploy
     print_runtime do
@@ -43,12 +52,13 @@ class Mrsk::Cli::Main < Mrsk::Cli::Base
   def details
     invoke "mrsk:cli:traefik:details"
     invoke "mrsk:cli:app:details"
+    invoke "mrsk:cli:accessory:details", [ "all" ]
   end
 
   desc "config", "Show combined config"
   def config
     run_locally do
-      pp MRSK.config.to_h
+      puts MRSK.config.to_h.to_yaml
     end
   end
 

data/lib/mrsk/cli/server.rb

@@ -3,6 +3,6 @@ require "mrsk/cli/base"
 class Mrsk::Cli::Server < Mrsk::Cli::Base
   desc "bootstrap", "Ensure Docker is installed on the servers"
   def bootstrap
-    on(MRSK.hosts) { execute "which docker || (apt-get update -y && apt-get install docker.io -y)" }
+    on(MRSK.hosts + MRSK.accessory_hosts) { execute "which docker || (apt-get update -y && apt-get install docker.io -y)" }
   end
 end

data/lib/mrsk/commander.rb

@@ -48,6 +48,10 @@ class Mrsk::Commander
     specific_hosts || config.accessories.collect(&:host)
   end
 
+  def accessory_names
+    config.accessories&.collect(&:name) || []
+  end
+
 
   def app
     @app ||= Mrsk::Commands::App.new(config)
@@ -70,7 +74,7 @@ class Mrsk::Commander
   end
 
   def accessory(name)
-    (@accessories ||= {})[name] ||= Mrsk::Commands::Accessory.new(config, name: name)
+    config.accessories.detect { |a| a.name == name }
   end
 
 

data/lib/mrsk/commands/accessory.rb

@@ -2,7 +2,7 @@ require "mrsk/commands/base"
 
 class Mrsk::Commands::Accessory < Mrsk::Commands::Base
   attr_reader :accessory_config
-  delegate :service_name, :image, :host, :port, :env_args, :volume_args, :label_args, to: :accessory_config
+  delegate :service_name, :image, :host, :port, :files, :directories, :env_args, :volume_args, :label_args, to: :accessory_config
 
   def initialize(config, name:)
     super(config)
@@ -46,6 +46,47 @@ class Mrsk::Commands::Accessory < Mrsk::Commands::Base
     ).join(" "), host: host
   end
 
+  def exec(*command, interactive: false)
+    docker :exec,
+      ("-it" if interactive),
+      *env_args,
+      *volume_args,
+      service_name,
+      *command
+  end
+
+  def run_exec(*command, interactive: false)
+    docker :run,
+      ("-it" if interactive),
+      "--rm",
+      *env_args,
+      *volume_args,
+      image,
+      *command
+  end
+
+  def bash(host:)
+    exec_over_ssh "bash", host: host
+  end
+
+  def ensure_local_file_present(local_file)
+    if !local_file.is_a?(StringIO) && !Pathname.new(local_file).exist?
+      raise "Missing file: #{local_file}"
+    end
+  end
+
+  def make_directory_for(remote_file)
+    make_directory Pathname.new(remote_file).dirname.to_s
+  end
+
+  def make_directory(path)
+    [ :mkdir, "-p", path ]
+  end
+
+  def remove_service_directory
+    [ :rm, "-rf", service_name ]
+  end
+
   def remove_container
     docker :container, :prune, "-f", *service_filter
   end
@@ -55,6 +96,10 @@ class Mrsk::Commands::Accessory < Mrsk::Commands::Base
   end
 
   private
+    def exec_over_ssh(*command, host:)
+      run_over_ssh run_exec(*command, interactive: true).join(" "), host: host
+    end
+
     def service_filter
       [ "--filter", "label=service=#{service_name}" ]
     end

data/lib/mrsk/commands/app.rb

@@ -60,6 +60,10 @@ class Mrsk::Commands::App < Mrsk::Commands::Base
       *command
   end
 
+  def exec_over_ssh(*command, host:)
+    run_over_ssh run_exec(*command, interactive: true).join(" "), host: host
+  end
+
   def follow_logs(host:, grep: nil)
     run_over_ssh pipe(
       current_container_id,
@@ -89,15 +93,15 @@ class Mrsk::Commands::App < Mrsk::Commands::Base
   end
 
   private
-    def exec_over_ssh(*command, host:)
-      run_over_ssh run_exec(*command, interactive: true).join(" "), host: host
-    end
-
     def service_filter
       [ "--filter", "label=service=#{config.service}" ]
     end
 
     def rails_master_key_arg
-      [ "-e", redact("RAILS_MASTER_KEY=#{config.master_key}") ]
+      if master_key = config.master_key
+        [ "-e", redact("RAILS_MASTER_KEY=#{master_key}") ]
+      else
+        []
+      end
     end
 end

data/lib/mrsk/commands/base.rb

@@ -16,6 +16,10 @@ module Mrsk::Commands
           .tap     { |commands| commands.pop } # Remove trailing combiner
       end
 
+      def chain(*commands)
+        combine *commands, by: ";"
+      end
+
       def pipe(*commands)
         combine *commands, by: "|"
       end

data/lib/mrsk/commands/builder/native/remote.rb

@@ -2,13 +2,13 @@ require "mrsk/commands/builder/native"
 
 class Mrsk::Commands::Builder::Native::Remote < Mrsk::Commands::Builder::Native
   def create
-    combine \
+    chain \
       create_context,
       create_buildx
   end
 
   def remove
-    combine \
+    chain \
       remove_context,
       remove_buildx
   end
@@ -25,7 +25,7 @@ class Mrsk::Commands::Builder::Native::Remote < Mrsk::Commands::Builder::Native
   end
 
   def info
-    combine \
+    chain \
       docker(:context, :ls),
       docker(:buildx, :ls)
   end

data/lib/mrsk/configuration/accessory.rb

@@ -43,8 +43,22 @@ class Mrsk::Configuration::Assessory
     argumentize_env_with_secrets env
   end
 
+  def files
+    specifics["files"]&.to_h do |local_to_remote_mapping|
+      local_file, remote_file = local_to_remote_mapping.split(":")
+      [ expand_local_file(local_file), expand_remote_file(remote_file) ]
+    end || {}
+  end
+
+  def directories
+    specifics["directories"]&.to_h do |host_to_container_mapping|
+      host_relative_path, container_path = host_to_container_mapping.split(":")
+      [ expand_host_path(host_relative_path), container_path ]
+    end || {}
+  end
+
   def volumes
-    specifics["volumes"] || []
+    specific_volumes + remote_files_as_volumes + remote_directories_as_volumes
   end
 
   def volume_args
@@ -57,4 +71,46 @@ class Mrsk::Configuration::Assessory
     def default_labels
       { "service" => service_name }
     end
+
+    def expand_local_file(local_file)
+      if local_file.end_with?("erb")
+        read_dynamic_file(local_file)
+      else
+        Pathname.new(File.expand_path(local_file)).to_s
+      end
+    end
+
+    def read_dynamic_file(local_file)
+      StringIO.new(ERB.new(IO.read(local_file)).result)
+    end
+
+    def expand_remote_file(remote_file)
+      service_name + remote_file
+    end
+
+    def specific_volumes
+      specifics["volumes"] || []
+    end
+
+    def remote_files_as_volumes
+      specifics["files"]&.collect do |local_to_remote_mapping|
+        _, remote_file = local_to_remote_mapping.split(":")
+        "#{service_data_directory + remote_file}:#{remote_file}"
+      end || []
+    end
+
+    def remote_directories_as_volumes
+      specifics["directories"]&.collect do |host_to_container_mapping|
+        host_relative_path, container_path = host_to_container_mapping.split(":")
+        [ expand_host_path(host_relative_path), container_path ].join(":")
+      end || []
+    end
+
+    def expand_host_path(host_relative_path)
+      "#{service_data_directory}/#{host_relative_path}"
+    end
+
+    def service_data_directory
+      "$PWD/#{service_name}"
+    end
 end

data/lib/mrsk/configuration.rb

@@ -15,7 +15,7 @@ class Mrsk::Configuration
     def create_from(base_config_file, destination: nil, version: "missing")
       new(load_config_file(base_config_file).tap do |config|
         if destination
-          config.merge! \
+          config.deep_merge! \
             load_config_file destination_config_file(base_config_file, destination)
         end
       end, version: version)
@@ -52,7 +52,7 @@ class Mrsk::Configuration
   end
 
   def accessories
-    @accessories ||= raw_config.accessories.keys.collect { |name| Mrsk::Configuration::Assessory.new(name, config: self) }
+    @accessories ||= raw_config.accessories&.keys&.collect { |name| Mrsk::Configuration::Assessory.new(name, config: self) } || []
   end
 
   def accessory(name)
@@ -115,7 +115,9 @@ class Mrsk::Configuration
   end
 
   def master_key
-    ENV["RAILS_MASTER_KEY"] || File.read(Pathname.new(File.expand_path("config/master.key")))
+    unless raw_config.skip_master_key
+      ENV["RAILS_MASTER_KEY"] || File.read(Pathname.new(File.expand_path("config/master.key")))
+    end
   end
 
   def to_h
@@ -130,7 +132,8 @@ class Mrsk::Configuration
       env_args: env_args,
       volume_args: volume_args,
       ssh_options: ssh_options,
-      builder: raw_config.builder
+      builder: raw_config.builder,
+      accessories: raw_config.accessories
     }.compact
   end
 

data/lib/mrsk/version.rb

@@ -1,3 +1,3 @@
 module Mrsk
-  VERSION = "0.2.0"
+  VERSION = "0.3.0"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: mrsk
 version: !ruby/object:Gem::Version
-  version: 0.2.0
+  version: 0.3.0
 platform: ruby
 authors:
 - David Heinemeier Hansson
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2023-01-23 00:00:00.000000000 Z
+date: 2023-01-27 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activesupport
@@ -113,7 +113,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.4.1
+rubygems_version: 3.4.5
 signing_key:
 specification_version: 4
 summary: Deploy Rails apps in containers to servers running Docker with zero downtime.