mrsk 0.0.3 → 0.2.0

data/lib/mrsk/commands/builder/multiarch/remote.rb

@@ -15,12 +15,20 @@ class Mrsk::Commands::Builder::Multiarch::Remote < Mrsk::Commands::Builder::Mult
   end
 
   private
+    def builder_name
+      super + "-remote"
+    end
+
+    def builder_name_with_arch(arch)
+      "#{builder_name}-#{arch}"
+    end
+
     def create_local_buildx
-      docker :buildx, :create, "--use", "--name", "mrsk", "mrsk-#{local["arch"]}", "--platform", "linux/#{local["arch"]}"
+      docker :buildx, :create, "--name", builder_name, builder_name_with_arch(local["arch"]), "--platform", "linux/#{local["arch"]}"
     end
 
     def append_remote_buildx
-      docker :buildx, :create, "--append", "--name", "mrsk", "mrsk-#{remote["arch"]}", "--platform", "linux/#{remote["arch"]}"
+      docker :buildx, :create, "--append", "--name", builder_name, builder_name_with_arch(remote["arch"]), "--platform", "linux/#{remote["arch"]}"
     end
 
     def create_contexts
@@ -30,7 +38,7 @@ class Mrsk::Commands::Builder::Multiarch::Remote < Mrsk::Commands::Builder::Mult
     end
 
     def create_context(arch, host)
-      docker :context, :create, "mrsk-#{arch}", "--description", "'MRSK #{arch} Native Host'", "--docker", "'host=#{host}'"
+      docker :context, :create, builder_name_with_arch(arch), "--description", "'#{builder_name} #{arch} native host'", "--docker", "'host=#{host}'"
     end
 
     def remove_contexts
@@ -40,7 +48,7 @@ class Mrsk::Commands::Builder::Multiarch::Remote < Mrsk::Commands::Builder::Mult
     end
 
     def remove_context(arch)
-      docker :context, :rm, "mrsk-#{arch}"
+      docker :context, :rm, builder_name_with_arch(arch)
     end
 
     def local

data/lib/mrsk/commands/builder/multiarch.rb

@@ -1,20 +1,23 @@
-require "mrsk/commands/base"
+require "mrsk/commands/builder/base"
 
-class Mrsk::Commands::Builder::Multiarch < Mrsk::Commands::Base
+class Mrsk::Commands::Builder::Multiarch < Mrsk::Commands::Builder::Base
   def create
-    docker :buildx, :create, "--use", "--name", "mrsk"
+    docker :buildx, :create, "--use", "--name", builder_name
   end
 
   def remove
-    docker :buildx, :rm, "mrsk"
+    docker :buildx, :rm, builder_name
   end
 
   def push
-    docker :buildx, :build, "--push", "--platform linux/amd64,linux/arm64", "-t", config.absolute_image, "."
-  end
-
-  def pull
-    docker :pull, config.absolute_image
+    docker :buildx, :build,
+      "--push",
+      "--platform", "linux/amd64,linux/arm64",
+      "--builder", builder_name,
+      "-t", config.absolute_image,
+      *build_args,
+      *build_secrets,
+      "."
   end
 
   def info
@@ -22,4 +25,9 @@ class Mrsk::Commands::Builder::Multiarch < Mrsk::Commands::Base
       docker(:context, :ls),
       docker(:buildx, :ls)
   end
+
+  private
+    def builder_name
+      "mrsk-#{config.service}-multiarch"
+    end
 end

data/lib/mrsk/commands/builder/native/remote.rb

@@ -0,0 +1,71 @@
+require "mrsk/commands/builder/native"
+
+class Mrsk::Commands::Builder::Native::Remote < Mrsk::Commands::Builder::Native
+  def create
+    combine \
+      create_context,
+      create_buildx
+  end
+
+  def remove
+    combine \
+      remove_context,
+      remove_buildx
+  end
+
+  def push
+    docker :buildx, :build,
+      "--push",
+      "--platform", platform,
+      "--builder", builder_name,
+      "-t", config.absolute_image,
+      *build_args,
+      *build_secrets,
+      "."
+  end
+
+  def info
+    combine \
+      docker(:context, :ls),
+      docker(:buildx, :ls)
+  end
+
+
+  private
+    def arch
+      config.builder["remote"]["arch"]
+    end
+
+    def host
+      config.builder["remote"]["host"]
+    end
+
+    def builder_name
+      "mrsk-#{config.service}-native-remote"
+    end
+
+    def builder_name_with_arch
+      "#{builder_name}-#{arch}"
+    end
+
+    def platform
+      "linux/#{arch}"
+    end
+
+    def create_context
+      docker :context, :create,
+        builder_name_with_arch, "--description", "'#{builder_name} #{arch} native host'", "--docker", "'host=#{host}'"
+    end
+
+    def remove_context
+      docker :context, :rm, builder_name_with_arch
+    end
+
+    def create_buildx
+      docker :buildx, :create, "--name", builder_name, builder_name_with_arch, "--platform", platform
+    end
+
+    def remove_buildx
+      docker :buildx, :rm, builder_name
+    end
+end

data/lib/mrsk/commands/builder/native.rb

@@ -1,6 +1,6 @@
-require "mrsk/commands/base"
+require "mrsk/commands/builder/base"
 
-class Mrsk::Commands::Builder::Native < Mrsk::Commands::Base
+class Mrsk::Commands::Builder::Native < Mrsk::Commands::Builder::Base
   def create
     # No-op on native
   end
@@ -11,14 +11,10 @@ class Mrsk::Commands::Builder::Native < Mrsk::Commands::Base
 
   def push
     combine \
-      docker(:build, "-t", config.absolute_image, "."),
+      docker(:build, "-t", *build_args, *build_secrets, config.absolute_image, "."),
       docker(:push, config.absolute_image)
   end
 
-  def pull
-    docker :pull, config.absolute_image
-  end
-
   def info
     # No-op on native
   end

data/lib/mrsk/commands/builder.rb

@@ -2,22 +2,21 @@ require "mrsk/commands/base"
 
 class Mrsk::Commands::Builder < Mrsk::Commands::Base
   delegate :create, :remove, :push, :pull, :info, to: :target
-  delegate :native?, :multiarch?, :remote?, to: :name
 
   def name
-    target.class.to_s.demodulize.downcase.inquiry
+    target.class.to_s.remove("Mrsk::Commands::Builder::").underscore
   end
 
   def target
     case
-    when config.builder.nil?
-      multiarch
-    when config.builder["multiarch"] == false
+    when config.builder && config.builder["multiarch"] == false
       native
-    when config.builder["local"] && config.builder["local"]
+    when config.builder && config.builder["local"] && config.builder["remote"]
       multiarch_remote
+    when config.builder && config.builder["remote"]
+      native_remote
     else
-      raise ArgumentError, "Builder configuration incorrect: #{config.builder.inspect}"
+      multiarch
     end
   end
 
@@ -25,6 +24,10 @@ class Mrsk::Commands::Builder < Mrsk::Commands::Base
     @native ||= Mrsk::Commands::Builder::Native.new(config)
   end
 
+  def native_remote
+    @native ||= Mrsk::Commands::Builder::Native::Remote.new(config)
+  end
+
   def multiarch
     @multiarch ||= Mrsk::Commands::Builder::Multiarch.new(config)
   end
@@ -35,5 +38,6 @@ class Mrsk::Commands::Builder < Mrsk::Commands::Base
 end
 
 require "mrsk/commands/builder/native"
+require "mrsk/commands/builder/native/remote"
 require "mrsk/commands/builder/multiarch"
 require "mrsk/commands/builder/multiarch/remote"

data/lib/mrsk/commands/traefik.rb

@@ -8,7 +8,8 @@ class Mrsk::Commands::Traefik < Mrsk::Commands::Base
       "-p 80:80",
       "-v /var/run/docker.sock:/var/run/docker.sock",
       "traefik",
-      "--providers.docker"
+      "--providers.docker",
+      "--log.level=DEBUG"
   end
 
   def start
@@ -23,8 +24,17 @@ class Mrsk::Commands::Traefik < Mrsk::Commands::Base
     docker :ps, "--filter", "name=traefik"
   end
 
-  def logs
-    docker :logs, "traefik", "-n", "100", "-t"
+  def logs(since: nil, lines: nil, grep: nil)
+    pipe \
+      docker(:logs, "traefik", (" --since #{since}" if since), (" -n #{lines}" if lines), "-t", "2>&1"),
+      ("grep '#{grep}'" if grep)
+  end
+
+  def follow_logs(host:, grep: nil)
+    run_over_ssh pipe(
+      docker(:logs, "traefik", "-t", "-n", "10", "-f", "2>&1"),
+      ("grep '#{grep}'" if grep)
+    ).join(" "), host: host
   end
 
   def remove_container

data/lib/mrsk/configuration/accessory.rb

@@ -0,0 +1,60 @@
+class Mrsk::Configuration::Assessory
+  delegate :argumentize, :argumentize_env_with_secrets, to: Mrsk::Utils
+
+  attr_accessor :name, :specifics
+
+  def initialize(name, config:)
+    @name, @config, @specifics = name.inquiry, config, config.raw_config["accessories"][name]
+  end
+
+  def service_name
+    "#{config.service}-#{name}"
+  end
+
+  def image
+    specifics["image"]
+  end
+
+  def host
+    specifics["host"] || raise(ArgumentError, "Missing host for accessory")
+  end
+
+  def port
+    if specifics["port"].to_s.include?(":")
+      specifics["port"]
+    else
+      "#{specifics["port"]}:#{specifics["port"]}"
+    end
+  end
+
+  def labels
+    default_labels.merge(specifics["labels"] || {})
+  end
+
+  def label_args
+    argumentize "--label", labels
+  end
+
+  def env
+    specifics["env"] || {}
+  end
+
+  def env_args
+    argumentize_env_with_secrets env
+  end
+
+  def volumes
+    specifics["volumes"] || []
+  end
+
+  def volume_args
+    argumentize "--volume", volumes
+  end
+
+  private
+    attr_accessor :config
+
+    def default_labels
+      { "service" => service_name }
+    end
+end

data/lib/mrsk/configuration/role.rb

@@ -1,5 +1,5 @@
 class Mrsk::Configuration::Role
-  delegate :argumentize, to: Mrsk::Configuration
+  delegate :argumentize, :argumentize_env_with_secrets, to: Mrsk::Utils
 
   attr_accessor :name
 
@@ -12,21 +12,33 @@ class Mrsk::Configuration::Role
   end
 
   def labels
-    if name.web?
-      default_labels.merge(traefik_labels).merge(custom_labels)
-    else
-      default_labels.merge(custom_labels)
-    end
+    default_labels.merge(traefik_labels).merge(custom_labels)
   end
 
   def label_args
     argumentize "--label", labels
   end
 
+  def env
+    if config.env && config.env["secret"]
+      merged_env_with_secrets
+    else
+      merged_env
+    end
+  end
+
+  def env_args
+    argumentize_env_with_secrets env
+  end
+
   def cmd
     specializations["cmd"]
   end
 
+  def running_traefik?
+    name.web? || specializations["traefik"]
+  end
+
   private
     attr_accessor :config
 
@@ -44,13 +56,17 @@ class Mrsk::Configuration::Role
     end
 
     def traefik_labels
-      {
-        "traefik.http.routers.#{config.service}.rule" => "'PathPrefix(`/`)'",
-        "traefik.http.services.#{config.service}.loadbalancer.healthcheck.path" => "/up",
-        "traefik.http.services.#{config.service}.loadbalancer.healthcheck.interval" => "1s",
-        "traefik.http.middlewares.#{config.service}.retry.attempts" => "3",
-        "traefik.http.middlewares.#{config.service}.retry.initialinterval" => "500ms"
-      }
+      if running_traefik?
+        {
+          "traefik.http.routers.#{config.service}.rule" => "'PathPrefix(`/`)'",
+          "traefik.http.services.#{config.service}.loadbalancer.healthcheck.path" => "/up",
+          "traefik.http.services.#{config.service}.loadbalancer.healthcheck.interval" => "1s",
+          "traefik.http.middlewares.#{config.service}.retry.attempts" => "3",
+          "traefik.http.middlewares.#{config.service}.retry.initialinterval" => "500ms"
+        }
+      else
+        {}
+      end
     end
 
     def custom_labels
@@ -64,7 +80,23 @@ class Mrsk::Configuration::Role
       if config.servers.is_a?(Array) || config.servers[name].is_a?(Array)
         { }
       else
-        config.servers[name].without("hosts")
+        config.servers[name].except("hosts")
+      end
+    end
+
+    def specialized_env
+      specializations["env"] || {}
+    end
+
+    def merged_env
+      config.env&.merge(specialized_env) || {}
+    end
+
+    # Secrets are stored in an array, which won't merge by default, so have to do it by hand.
+    def merged_env_with_secrets
+      merged_env.tap do |new_env|
+        new_env["secret"] = Array(config.env["secret"]) + Array(specialized_env["secret"])
+        new_env["clear"]  = (Array(config.env["clear"] || config.env) + Array(specialized_env["clear"] || specialized_env)).uniq
       end
     end
 end

data/lib/mrsk/configuration.rb

@@ -1,26 +1,44 @@
 require "active_support/ordered_options"
 require "active_support/core_ext/string/inquiry"
+require "active_support/core_ext/module/delegation"
+require "pathname"
 require "erb"
+require "mrsk/utils"
 
 class Mrsk::Configuration
-  delegate :service, :image, :servers, :env, :labels, :registry, :builder, to: :config, allow_nil: true
+  delegate :service, :image, :servers, :env, :labels, :registry, :builder, to: :raw_config, allow_nil: true
+  delegate :argumentize, :argumentize_env_with_secrets, to: Mrsk::Utils
+
+  attr_accessor :raw_config
 
   class << self
-    def load_file(file)
-      if file.exist?
-        new YAML.load(ERB.new(IO.read(file)).result).symbolize_keys
-      else
-        raise "Configuration file not found in #{file}"
-      end
+    def create_from(base_config_file, destination: nil, version: "missing")
+      new(load_config_file(base_config_file).tap do |config|
+        if destination
+          config.merge! \
+            load_config_file destination_config_file(base_config_file, destination)
+        end
+      end, version: version)
     end
 
-    def argumentize(argument, attributes)
-      attributes.flat_map { |k, v| [ argument, "#{k}=#{v}" ] }
-    end
+    private
+      def load_config_file(file)
+        if file.exist?
+          YAML.load(ERB.new(IO.read(file)).result).symbolize_keys
+        else
+          raise "Configuration file not found in #{file}"
+        end
+      end
+
+      def destination_config_file(base_config_file, destination)
+        dir, basename = base_config_file.split
+        dir.join basename.to_s.remove(".yml") + ".#{destination}.yml"
+      end
   end
 
-  def initialize(config, validate: true)
-    @config = ActiveSupport::InheritableOptions.new(config)
+  def initialize(raw_config, version: "missing", validate: true)
+    @raw_config = ActiveSupport::InheritableOptions.new(raw_config)
+    @version = version
     ensure_required_keys_present if validate
   end
 
@@ -33,36 +51,34 @@ class Mrsk::Configuration
     roles.detect { |r| r.name == name.to_s }
   end
 
-  def hosts
-    hosts =
-      case
-      when ENV["HOSTS"]
-        ENV["HOSTS"].split(",")
-      when ENV["ROLES"]
-        role_names = ENV["ROLES"].split(",")
-        roles.select { |r| role_names.include?(r.name) }.flat_map(&:hosts)
-      else
-        roles.flat_map(&:hosts)
-      end
+  def accessories
+    @accessories ||= raw_config.accessories.keys.collect { |name| Mrsk::Configuration::Assessory.new(name, config: self) }
+  end
 
-      if hosts.any?
-        hosts
-      else
-        raise ArgumentError, "No hosts found"
-      end
+  def accessory(name)
+    accessories.detect { |a| a.name == name.to_s }
+  end
+
+
+  def all_hosts
+    roles.flat_map(&:hosts)
   end
 
-  def primary_host
+  def primary_web_host
     role(:web).hosts.first
   end
 
+  def traefik_hosts
+    roles.select(&:running_traefik?).flat_map(&:hosts)
+  end
+
 
   def version
-    @version ||= ENV["VERSION"] || `git rev-parse HEAD`.strip
+    @version
   end
 
   def repository
-    [ config.registry["server"], image ].compact.join("/")
+    [ raw_config.registry["server"], image ].compact.join("/")
   end
 
   def absolute_image
@@ -75,15 +91,23 @@ class Mrsk::Configuration
 
 
   def env_args
-    if config.env.present?
-      self.class.argumentize "-e", config.env
+    if raw_config.env.present?
+      argumentize_env_with_secrets(raw_config.env)
+    else
+      []
+    end
+  end
+
+  def volume_args
+    if raw_config.volumes.present?
+      argumentize "--volume", raw_config.volumes
     else
       []
     end
   end
 
   def ssh_user
-    config.ssh_user || "root"
+    raw_config.ssh_user || "root"
   end
 
   def ssh_options
@@ -91,26 +115,45 @@ class Mrsk::Configuration
   end
 
   def master_key
-    ENV["RAILS_MASTER_KEY"] || File.read(Rails.root.join("config/master.key"))
+    ENV["RAILS_MASTER_KEY"] || File.read(Pathname.new(File.expand_path("config/master.key")))
   end
 
+  def to_h
+    {
+      roles: role_names,
+      hosts: all_hosts,
+      primary_host: primary_web_host,
+      version: version,
+      repository: repository,
+      absolute_image: absolute_image,
+      service_with_version: service_with_version,
+      env_args: env_args,
+      volume_args: volume_args,
+      ssh_options: ssh_options,
+      builder: raw_config.builder
+    }.compact
+  end
 
-  private
-    attr_accessor :config
 
+  private
     def ensure_required_keys_present
-      %i[ service image registry ].each do |key|
-        raise ArgumentError, "Missing required configuration for #{key}" unless config[key].present?
+      %i[ service image registry servers ].each do |key|
+        raise ArgumentError, "Missing required configuration for #{key}" unless raw_config[key].present?
       end
 
-      %w[ username password ].each do |key|
-        raise ArgumentError, "Missing required configuration for registry/#{key}" unless config.registry[key].present?
-      end      
+      if raw_config.registry["username"].blank?
+        raise ArgumentError, "You must specify a username for the registry in config/deploy.yml"
+      end
+
+      if raw_config.registry["password"].blank?
+        raise ArgumentError, "You must specify a password for the registry in config/deploy.yml (or set the ENV variable if that's used)"
+      end
     end
 
     def role_names
-      config.servers.is_a?(Array) ? [ "web" ] : config.servers.keys.sort
+      raw_config.servers.is_a?(Array) ? [ "web" ] : raw_config.servers.keys.sort
     end
 end
 
 require "mrsk/configuration/role"
+require "mrsk/configuration/accessory"

data/lib/mrsk/sshkit_with_ext.rb

@@ -0,0 +1,12 @@
+require "sshkit"
+require "sshkit/dsl"
+
+class SSHKit::Backend::Abstract
+  def capture_with_info(*args)
+    capture(*args, verbosity: Logger::INFO)
+  end
+
+  def puts_by_host(host, output, type: "App")
+    puts "#{type} Host: #{host}\n#{output}\n\n"
+  end
+end

data/lib/mrsk/utils.rb

@@ -0,0 +1,29 @@
+module Mrsk::Utils
+  extend self
+
+  # Return a list of shell arguments using the same named argument against the passed attributes (hash or array).
+  def argumentize(argument, attributes, redacted: false)
+    Array(attributes).flat_map do |k, v|
+      if v.present?
+        [ argument, redacted ? redact("#{k}=#{v}") : "#{k}=#{v}" ]
+      else
+        [ argument, k ]
+      end
+    end
+  end
+
+  # Return a list of shell arguments using the same named argument against the passed attributes,
+  # but redacts and expands secrets.
+  def argumentize_env_with_secrets(env)
+    if (secrets = env["secret"]).present?
+      argumentize("-e", secrets.to_h { |key| [ key, ENV.fetch(key) ] }, redacted: true) + argumentize("-e", env["clear"])
+    else
+      argumentize "-e", env
+    end
+  end
+
+  # Copied from SSHKit::Backend::Abstract#redact to be available inside Commands classes
+  def redact(arg) # Used in execute_command to hide redact() args a user passes in
+    arg.to_s.extend(SSHKit::Redaction) # to_s due to our inability to extend Integer, etc
+  end
+end

data/lib/mrsk/version.rb

@@ -1,3 +1,3 @@
 module Mrsk
-  VERSION = "0.0.3"
+  VERSION = "0.2.0"
 end

data/lib/mrsk.rb

@@ -2,5 +2,4 @@ module Mrsk
 end
 
 require "mrsk/version"
-require "mrsk/engine"
 require "mrsk/commander"