mrsk 0.1.0 → 0.3.0

data/lib/mrsk/commands/app.rb

@@ -8,8 +8,9 @@ class Mrsk::Commands::App < Mrsk::Commands::Base
       "-d",
       "--restart unless-stopped",
       "--name", config.service_with_version,
-      "-e", redact("RAILS_MASTER_KEY=#{config.master_key}"),
-      *config.env_args,
+      *rails_master_key_arg,
+      *role.env_args,
+      *config.volume_args,
       *role.label_args,
       config.absolute_image,
       role.cmd
@@ -19,29 +20,64 @@ class Mrsk::Commands::App < Mrsk::Commands::Base
     docker :start, "#{config.service}-#{version}"
   end
 
+  def current_container_id
+    docker :ps, "-q", *service_filter
+  end
+
   def stop
-    [ "docker ps -q #{service_filter.join(" ")} | xargs docker stop" ]
+    pipe current_container_id, "xargs docker stop"
   end
 
   def info
     docker :ps, *service_filter
   end
 
-  def logs
-    [ "docker ps -q #{service_filter.join(" ")} | xargs docker logs -n 100 -t" ]
+  def logs(since: nil, lines: nil, grep: nil)
+    pipe \
+      current_container_id,
+      "xargs docker logs#{" --since #{since}" if since}#{" -n #{lines}" if lines} -t 2>&1",
+      ("grep '#{grep}'" if grep)
   end
 
   def exec(*command, interactive: false)
     docker :exec,
       ("-it" if interactive),
-      "-e", redact("RAILS_MASTER_KEY=#{config.master_key}"),
+      *rails_master_key_arg,
       *config.env_args,
+      *config.volume_args,
       config.service_with_version,
       *command
   end
 
-  def console(host: config.primary_host)
-    "ssh -t #{config.ssh_user}@#{host} '#{exec("bin/rails", "c", interactive: true).join(" ")}'"
+  def run_exec(*command, interactive: false)
+    docker :run,
+      ("-it" if interactive),
+      "--rm",
+      *rails_master_key_arg,
+      *config.env_args,
+      *config.volume_args,
+      config.absolute_image,
+      *command
+  end
+
+  def exec_over_ssh(*command, host:)
+    run_over_ssh run_exec(*command, interactive: true).join(" "), host: host
+  end
+
+  def follow_logs(host:, grep: nil)
+    run_over_ssh pipe(
+      current_container_id,
+      "xargs docker logs -t -n 10 -f 2>&1",
+      ("grep '#{grep}'" if grep)
+    ).join(" "), host: host
+  end
+
+  def console(host:)
+    exec_over_ssh "bin/rails", "c", host: host
+  end
+
+  def bash(host:)
+    exec_over_ssh "bash", host: host
   end
 
   def list_containers
@@ -60,4 +96,12 @@ class Mrsk::Commands::App < Mrsk::Commands::Base
     def service_filter
       [ "--filter", "label=service=#{config.service}" ]
     end
+
+    def rails_master_key_arg
+      if master_key = config.master_key
+        [ "-e", redact("RAILS_MASTER_KEY=#{master_key}") ]
+      else
+        []
+      end
+    end
 end

data/lib/mrsk/commands/base.rb

@@ -1,7 +1,7 @@
-require "sshkit"
-
 module Mrsk::Commands
   class Base
+    delegate :redact, to: Mrsk::Utils
+
     attr_accessor :config
 
     def initialize(config)
@@ -9,19 +9,27 @@ module Mrsk::Commands
     end
 
     private
-      def combine(*commands)
+      def combine(*commands, by: "&&")
         commands
-          .collect { |command| command + [ "&&" ] }.flatten # Join commands with &&
-          .tap     { |commands| commands.pop } # Remove trailing &&
+          .compact
+          .collect { |command| Array(command) + [ by ] }.flatten # Join commands
+          .tap     { |commands| commands.pop } # Remove trailing combiner
+      end
+
+      def chain(*commands)
+        combine *commands, by: ";"
+      end
+
+      def pipe(*commands)
+        combine *commands, by: "|"
       end
 
       def docker(*args)
         args.compact.unshift :docker
       end
 
-      # Copied from SSHKit::Backend::Abstract#redact to be available inside Commands classes
-      def redact(arg) # Used in execute_command to hide redact() args a user passes in
-        arg.to_s.extend(SSHKit::Redaction) # to_s due to our inability to extend Integer, etc
-      end      
+      def run_over_ssh(command, host:)
+        "ssh -t #{config.ssh_user}@#{host} '#{command}'"
+      end
   end
 end

data/lib/mrsk/commands/builder/base.rb

@@ -0,0 +1,26 @@
+require "mrsk/commands/base"
+
+class Mrsk::Commands::Builder::Base < Mrsk::Commands::Base
+  delegate :argumentize, to: Mrsk::Utils
+
+  def pull
+    docker :pull, config.absolute_image
+  end
+
+  def build_args
+    argumentize "--build-arg", args, redacted: true
+  end
+
+  def build_secrets
+    argumentize "--secret", secrets.collect { |secret| [ "id", secret ] }
+  end
+
+  private
+    def args
+      (config.builder && config.builder["args"]) || {}
+    end
+
+    def secrets
+      (config.builder && config.builder["secrets"]) || []
+    end
+end

data/lib/mrsk/commands/builder/multiarch/remote.rb

@@ -15,8 +15,16 @@ class Mrsk::Commands::Builder::Multiarch::Remote < Mrsk::Commands::Builder::Mult
   end
 
   private
+    def builder_name
+      super + "-remote"
+    end
+
+    def builder_name_with_arch(arch)
+      "#{builder_name}-#{arch}"
+    end
+
     def create_local_buildx
-      docker :buildx, :create, "--use", "--name", builder_name, builder_name_with_arch(local["arch"]), "--platform", "linux/#{local["arch"]}"
+      docker :buildx, :create, "--name", builder_name, builder_name_with_arch(local["arch"]), "--platform", "linux/#{local["arch"]}"
     end
 
     def append_remote_buildx
@@ -50,9 +58,4 @@ class Mrsk::Commands::Builder::Multiarch::Remote < Mrsk::Commands::Builder::Mult
     def remote
       config.builder["remote"]
     end
-
-    private
-      def builder_name_with_arch(arch)
-        "#{builder_name}-#{arch}"
-      end
 end

data/lib/mrsk/commands/builder/multiarch.rb

@@ -1,6 +1,6 @@
-require "mrsk/commands/base"
+require "mrsk/commands/builder/base"
 
-class Mrsk::Commands::Builder::Multiarch < Mrsk::Commands::Base
+class Mrsk::Commands::Builder::Multiarch < Mrsk::Commands::Builder::Base
   def create
     docker :buildx, :create, "--use", "--name", builder_name
   end
@@ -10,11 +10,14 @@ class Mrsk::Commands::Builder::Multiarch < Mrsk::Commands::Base
   end
 
   def push
-    docker :buildx, :build, "--push", "--platform linux/amd64,linux/arm64", "-t", config.absolute_image, "."
-  end
-
-  def pull
-    docker :pull, config.absolute_image
+    docker :buildx, :build,
+      "--push",
+      "--platform", "linux/amd64,linux/arm64",
+      "--builder", builder_name,
+      "-t", config.absolute_image,
+      *build_args,
+      *build_secrets,
+      "."
   end
 
   def info
@@ -25,6 +28,6 @@ class Mrsk::Commands::Builder::Multiarch < Mrsk::Commands::Base
 
   private
     def builder_name
-      "mrsk-#{config.service}"
+      "mrsk-#{config.service}-multiarch"
     end
 end

data/lib/mrsk/commands/builder/native/remote.rb

@@ -0,0 +1,71 @@
+require "mrsk/commands/builder/native"
+
+class Mrsk::Commands::Builder::Native::Remote < Mrsk::Commands::Builder::Native
+  def create
+    chain \
+      create_context,
+      create_buildx
+  end
+
+  def remove
+    chain \
+      remove_context,
+      remove_buildx
+  end
+
+  def push
+    docker :buildx, :build,
+      "--push",
+      "--platform", platform,
+      "--builder", builder_name,
+      "-t", config.absolute_image,
+      *build_args,
+      *build_secrets,
+      "."
+  end
+
+  def info
+    chain \
+      docker(:context, :ls),
+      docker(:buildx, :ls)
+  end
+
+
+  private
+    def arch
+      config.builder["remote"]["arch"]
+    end
+
+    def host
+      config.builder["remote"]["host"]
+    end
+
+    def builder_name
+      "mrsk-#{config.service}-native-remote"
+    end
+
+    def builder_name_with_arch
+      "#{builder_name}-#{arch}"
+    end
+
+    def platform
+      "linux/#{arch}"
+    end
+
+    def create_context
+      docker :context, :create,
+        builder_name_with_arch, "--description", "'#{builder_name} #{arch} native host'", "--docker", "'host=#{host}'"
+    end
+
+    def remove_context
+      docker :context, :rm, builder_name_with_arch
+    end
+
+    def create_buildx
+      docker :buildx, :create, "--name", builder_name, builder_name_with_arch, "--platform", platform
+    end
+
+    def remove_buildx
+      docker :buildx, :rm, builder_name
+    end
+end

data/lib/mrsk/commands/builder/native.rb

@@ -1,6 +1,6 @@
-require "mrsk/commands/base"
+require "mrsk/commands/builder/base"
 
-class Mrsk::Commands::Builder::Native < Mrsk::Commands::Base
+class Mrsk::Commands::Builder::Native < Mrsk::Commands::Builder::Base
   def create
     # No-op on native
   end
@@ -11,14 +11,10 @@ class Mrsk::Commands::Builder::Native < Mrsk::Commands::Base
 
   def push
     combine \
-      docker(:build, "-t", config.absolute_image, "."),
+      docker(:build, "-t", *build_args, *build_secrets, config.absolute_image, "."),
       docker(:push, config.absolute_image)
   end
 
-  def pull
-    docker :pull, config.absolute_image
-  end
-
   def info
     # No-op on native
   end

data/lib/mrsk/commands/builder.rb

@@ -2,22 +2,21 @@ require "mrsk/commands/base"
 
 class Mrsk::Commands::Builder < Mrsk::Commands::Base
   delegate :create, :remove, :push, :pull, :info, to: :target
-  delegate :native?, :multiarch?, :remote?, to: :name
 
   def name
-    target.class.to_s.demodulize.downcase.inquiry
+    target.class.to_s.remove("Mrsk::Commands::Builder::").underscore
   end
 
   def target
     case
-    when config.builder.nil?
-      multiarch
-    when config.builder["multiarch"] == false
+    when config.builder && config.builder["multiarch"] == false
       native
-    when config.builder["local"] && config.builder["local"]
+    when config.builder && config.builder["local"] && config.builder["remote"]
       multiarch_remote
+    when config.builder && config.builder["remote"]
+      native_remote
     else
-      raise ArgumentError, "Builder configuration incorrect: #{config.builder.inspect}"
+      multiarch
     end
   end
 
@@ -25,6 +24,10 @@ class Mrsk::Commands::Builder < Mrsk::Commands::Base
     @native ||= Mrsk::Commands::Builder::Native.new(config)
   end
 
+  def native_remote
+    @native ||= Mrsk::Commands::Builder::Native::Remote.new(config)
+  end
+
   def multiarch
     @multiarch ||= Mrsk::Commands::Builder::Multiarch.new(config)
   end
@@ -35,5 +38,6 @@ class Mrsk::Commands::Builder < Mrsk::Commands::Base
 end
 
 require "mrsk/commands/builder/native"
+require "mrsk/commands/builder/native/remote"
 require "mrsk/commands/builder/multiarch"
 require "mrsk/commands/builder/multiarch/remote"

data/lib/mrsk/commands/traefik.rb

@@ -8,7 +8,8 @@ class Mrsk::Commands::Traefik < Mrsk::Commands::Base
       "-p 80:80",
       "-v /var/run/docker.sock:/var/run/docker.sock",
       "traefik",
-      "--providers.docker"
+      "--providers.docker",
+      "--log.level=DEBUG"
   end
 
   def start
@@ -23,8 +24,17 @@ class Mrsk::Commands::Traefik < Mrsk::Commands::Base
     docker :ps, "--filter", "name=traefik"
   end
 
-  def logs
-    docker :logs, "traefik", "-n", "100", "-t"
+  def logs(since: nil, lines: nil, grep: nil)
+    pipe \
+      docker(:logs, "traefik", (" --since #{since}" if since), (" -n #{lines}" if lines), "-t", "2>&1"),
+      ("grep '#{grep}'" if grep)
+  end
+
+  def follow_logs(host:, grep: nil)
+    run_over_ssh pipe(
+      docker(:logs, "traefik", "-t", "-n", "10", "-f", "2>&1"),
+      ("grep '#{grep}'" if grep)
+    ).join(" "), host: host
   end
 
   def remove_container

data/lib/mrsk/configuration/accessory.rb

@@ -0,0 +1,116 @@
+class Mrsk::Configuration::Assessory
+  delegate :argumentize, :argumentize_env_with_secrets, to: Mrsk::Utils
+
+  attr_accessor :name, :specifics
+
+  def initialize(name, config:)
+    @name, @config, @specifics = name.inquiry, config, config.raw_config["accessories"][name]
+  end
+
+  def service_name
+    "#{config.service}-#{name}"
+  end
+
+  def image
+    specifics["image"]
+  end
+
+  def host
+    specifics["host"] || raise(ArgumentError, "Missing host for accessory")
+  end
+
+  def port
+    if specifics["port"].to_s.include?(":")
+      specifics["port"]
+    else
+      "#{specifics["port"]}:#{specifics["port"]}"
+    end
+  end
+
+  def labels
+    default_labels.merge(specifics["labels"] || {})
+  end
+
+  def label_args
+    argumentize "--label", labels
+  end
+
+  def env
+    specifics["env"] || {}
+  end
+
+  def env_args
+    argumentize_env_with_secrets env
+  end
+
+  def files
+    specifics["files"]&.to_h do |local_to_remote_mapping|
+      local_file, remote_file = local_to_remote_mapping.split(":")
+      [ expand_local_file(local_file), expand_remote_file(remote_file) ]
+    end || {}
+  end
+
+  def directories
+    specifics["directories"]&.to_h do |host_to_container_mapping|
+      host_relative_path, container_path = host_to_container_mapping.split(":")
+      [ expand_host_path(host_relative_path), container_path ]
+    end || {}
+  end
+
+  def volumes
+    specific_volumes + remote_files_as_volumes + remote_directories_as_volumes
+  end
+
+  def volume_args
+    argumentize "--volume", volumes
+  end
+
+  private
+    attr_accessor :config
+
+    def default_labels
+      { "service" => service_name }
+    end
+
+    def expand_local_file(local_file)
+      if local_file.end_with?("erb")
+        read_dynamic_file(local_file)
+      else
+        Pathname.new(File.expand_path(local_file)).to_s
+      end
+    end
+
+    def read_dynamic_file(local_file)
+      StringIO.new(ERB.new(IO.read(local_file)).result)
+    end
+
+    def expand_remote_file(remote_file)
+      service_name + remote_file
+    end
+
+    def specific_volumes
+      specifics["volumes"] || []
+    end
+
+    def remote_files_as_volumes
+      specifics["files"]&.collect do |local_to_remote_mapping|
+        _, remote_file = local_to_remote_mapping.split(":")
+        "#{service_data_directory + remote_file}:#{remote_file}"
+      end || []
+    end
+
+    def remote_directories_as_volumes
+      specifics["directories"]&.collect do |host_to_container_mapping|
+        host_relative_path, container_path = host_to_container_mapping.split(":")
+        [ expand_host_path(host_relative_path), container_path ].join(":")
+      end || []
+    end
+
+    def expand_host_path(host_relative_path)
+      "#{service_data_directory}/#{host_relative_path}"
+    end
+
+    def service_data_directory
+      "$PWD/#{service_name}"
+    end
+end

data/lib/mrsk/configuration/role.rb

@@ -1,5 +1,5 @@
 class Mrsk::Configuration::Role
-  delegate :argumentize, to: Mrsk::Configuration
+  delegate :argumentize, :argumentize_env_with_secrets, to: Mrsk::Utils
 
   attr_accessor :name
 
@@ -12,21 +12,33 @@ class Mrsk::Configuration::Role
   end
 
   def labels
-    if name.web?
-      default_labels.merge(traefik_labels).merge(custom_labels)
-    else
-      default_labels.merge(custom_labels)
-    end
+    default_labels.merge(traefik_labels).merge(custom_labels)
   end
 
   def label_args
     argumentize "--label", labels
   end
 
+  def env
+    if config.env && config.env["secret"]
+      merged_env_with_secrets
+    else
+      merged_env
+    end
+  end
+
+  def env_args
+    argumentize_env_with_secrets env
+  end
+
   def cmd
     specializations["cmd"]
   end
 
+  def running_traefik?
+    name.web? || specializations["traefik"]
+  end
+
   private
     attr_accessor :config
 
@@ -44,13 +56,17 @@ class Mrsk::Configuration::Role
     end
 
     def traefik_labels
-      {
-        "traefik.http.routers.#{config.service}.rule" => "'PathPrefix(`/`)'",
-        "traefik.http.services.#{config.service}.loadbalancer.healthcheck.path" => "/up",
-        "traefik.http.services.#{config.service}.loadbalancer.healthcheck.interval" => "1s",
-        "traefik.http.middlewares.#{config.service}.retry.attempts" => "3",
-        "traefik.http.middlewares.#{config.service}.retry.initialinterval" => "500ms"
-      }
+      if running_traefik?
+        {
+          "traefik.http.routers.#{config.service}.rule" => "'PathPrefix(`/`)'",
+          "traefik.http.services.#{config.service}.loadbalancer.healthcheck.path" => "/up",
+          "traefik.http.services.#{config.service}.loadbalancer.healthcheck.interval" => "1s",
+          "traefik.http.middlewares.#{config.service}.retry.attempts" => "3",
+          "traefik.http.middlewares.#{config.service}.retry.initialinterval" => "500ms"
+        }
+      else
+        {}
+      end
     end
 
     def custom_labels
@@ -64,7 +80,23 @@ class Mrsk::Configuration::Role
       if config.servers.is_a?(Array) || config.servers[name].is_a?(Array)
         { }
       else
-        config.servers[name].without("hosts")
+        config.servers[name].except("hosts")
+      end
+    end
+
+    def specialized_env
+      specializations["env"] || {}
+    end
+
+    def merged_env
+      config.env&.merge(specialized_env) || {}
+    end
+
+    # Secrets are stored in an array, which won't merge by default, so have to do it by hand.
+    def merged_env_with_secrets
+      merged_env.tap do |new_env|
+        new_env["secret"] = Array(config.env["secret"]) + Array(specialized_env["secret"])
+        new_env["clear"]  = (Array(config.env["clear"] || config.env) + Array(specialized_env["clear"] || specialized_env)).uniq
       end
     end
 end