tomo 0.1.0

data/lib/tomo/runtime/execution_plan.rb

@@ -0,0 +1,107 @@
+require "forwardable"
+
+module Tomo
+  class Runtime
+    class ExecutionPlan
+      extend Forwardable
+
+      def_delegators :@task_runner, :paths, :settings
+
+      attr_reader :applicable_hosts
+
+      def initialize(tasks:, hosts:, task_filter:, task_runner:)
+        @hosts = hosts
+        @tasks = tasks
+        @task_runner = task_runner
+        @plan = build_plan(task_filter)
+        @applicable_hosts = gather_applicable_hosts
+        @thread_pool = build_thread_pool
+        freeze
+        validate_tasks!
+      end
+
+      def applicable_hosts_sentence
+        return "no hosts" if applicable_hosts.empty?
+
+        case applicable_hosts.length
+        when 1 then applicable_hosts.first.to_s
+        when 2 then applicable_hosts.map(&:to_s).join(" and ")
+        else
+          "#{applicable_hosts.first} and "\
+          "#{applicable_hosts.length - 1} other hosts"
+        end
+      end
+
+      def execute
+        open_connections do |remotes|
+          plan.each do |steps|
+            steps.each do |step|
+              step.execute(thread_pool: thread_pool, remotes: remotes)
+            end
+            thread_pool.run_to_completion
+          end
+        end
+        self
+      end
+
+      private
+
+      attr_reader :tasks, :hosts, :plan, :task_runner, :thread_pool
+
+      def validate_tasks!
+        plan.each do |steps|
+          steps.each do |step|
+            step.applicable_tasks.each do |task|
+              task_runner.validate_task!(task)
+            end
+          end
+        end
+      end
+
+      def open_connections
+        remotes = applicable_hosts.each_with_object({}) do |host, opened|
+          thread_pool.post(host) do |thr_host|
+            opened[thr_host] = task_runner.connect(thr_host)
+          end
+        end
+        thread_pool.run_to_completion
+        yield(remotes)
+      ensure
+        (remotes || {}).values.each(&:close)
+      end
+
+      def build_plan(task_filter)
+        tasks.each_with_object([]) do |task, result|
+          steps = hosts.map do |host|
+            HostExecutionStep.new(
+              tasks: task, host: host,
+              task_filter: task_filter, task_runner: task_runner
+            )
+          end
+          steps.reject!(&:empty?)
+          result << steps unless steps.empty?
+        end
+      end
+
+      def gather_applicable_hosts
+        plan.each_with_object([]) do |steps, result|
+          steps.each do |step|
+            result.push(*step.applicable_hosts)
+          end
+        end.uniq
+      end
+
+      def build_thread_pool
+        if plan.map(&:length).max.to_i > 1
+          ConcurrentRubyThreadPool.new(concurrency)
+        else
+          InlineThreadPool.new
+        end
+      end
+
+      def concurrency
+        [settings[:concurrency].to_i, 1].max
+      end
+    end
+  end
+end

data/lib/tomo/runtime/host_execution_step.rb

@@ -0,0 +1,49 @@
+module Tomo
+  class Runtime
+    class HostExecutionStep
+      attr_reader :applicable_hosts, :applicable_tasks
+
+      def initialize(tasks:, host:, task_filter:, task_runner:)
+        tasks = Array(tasks).flatten
+        @host = host
+        @task_runner = task_runner
+        @applicable_tasks = task_filter.filter(tasks, host: @host).freeze
+        @applicable_hosts = compute_applicable_hosts
+        freeze
+      end
+
+      def empty?
+        applicable_tasks.empty?
+      end
+
+      def execute(thread_pool:, remotes:)
+        return if applicable_tasks.empty?
+
+        thread_pool.post do
+          applicable_tasks.each do |task|
+            break if thread_pool.failure?
+
+            task_host = task.is_a?(PrivilegedTask) ? host.as_privileged : host
+            remote = remotes[task_host]
+            task_runner.run(task: task, remote: remote)
+          end
+        end
+      end
+
+      private
+
+      attr_reader :host, :task_runner
+
+      def compute_applicable_hosts
+        priv_tasks, normal_tasks = applicable_tasks.partition do |task|
+          task.is_a?(PrivilegedTask)
+        end
+
+        hosts = []
+        hosts << host if normal_tasks.any?
+        hosts << host.as_privileged if priv_tasks.any?
+        hosts.uniq.freeze
+      end
+    end
+  end
+end

data/lib/tomo/runtime/inline_thread_pool.rb

@@ -0,0 +1,27 @@
+module Tomo
+  class Runtime
+    class InlineThreadPool
+      def post(*args)
+        return if failure?
+
+        yield(*args)
+        nil
+      rescue StandardError => e
+        self.failure = e
+        nil
+      end
+
+      def run_to_completion
+        raise failure if failure?
+      end
+
+      def failure?
+        !!failure
+      end
+
+      private
+
+      attr_accessor :failure
+    end
+  end
+end

data/lib/tomo/runtime/privileged_task.rb

@@ -0,0 +1,6 @@
+module Tomo
+  class Runtime
+    module PrivilegedTask
+    end
+  end
+end

data/lib/tomo/runtime/settings_interpolation.rb

@@ -0,0 +1,55 @@
+module Tomo
+  class Runtime
+    class SettingsInterpolation
+      def self.interpolate(settings)
+        new(settings).call
+      end
+
+      def initialize(settings)
+        @settings = symbolize(settings)
+      end
+
+      def call
+        hash = Hash[settings.keys.map { |name| [name, fetch(name)] }]
+        dump_settings(hash) if Tomo.debug?
+        hash
+      end
+
+      private
+
+      attr_reader :settings
+
+      def fetch(name, stack=[])
+        raise_circular_dependency_error(name, stack) if stack.include?(name)
+        value = settings.fetch(name)
+        return value unless value.is_a?(String)
+
+        value.gsub(/%<(\w+)>/) do
+          fetch(Regexp.last_match[1].to_sym, stack + [name])
+        end
+      end
+
+      def raise_circular_dependency_error(name, stack)
+        dependencies = [*stack, name].join(" -> ")
+        raise "Circular dependency detected in settings: #{dependencies}"
+      end
+
+      def symbolize(hash)
+        hash.each_with_object({}) do |(key, value), symbolized|
+          symbolized[key.to_sym] = value
+        end
+      end
+
+      def dump_settings(hash)
+        key_len = hash.keys.map(&:to_s).map(&:length).max
+        dump = "Settings: {\n"
+        hash.to_a.sort_by(&:first).each do |key, value|
+          justified_key = "#{key}:".ljust(key_len + 1)
+          dump << "  #{justified_key} #{value.inspect},\n"
+        end
+        dump << "}"
+        Tomo.logger.debug(dump)
+      end
+    end
+  end
+end

data/lib/tomo/runtime/settings_required_error.rb

@@ -0,0 +1,33 @@
+module Tomo
+  class Runtime
+    class SettingsRequiredError < Tomo::Error
+      attr_accessor :command_name, :settings, :task
+
+      def to_console
+        <<~ERROR
+          The #{yellow(task)} task requires #{settings_sentence}
+
+          Settings can be specified in #{blue(DEFAULT_CONFIG_PATH)}, or by running tomo
+          with the #{blue('-s')} option. For example:
+
+            #{blue("tomo -s #{settings.first}=foo")}
+
+          You can also use environment variables:
+
+            #{blue("TOMO_#{settings.first.upcase}=foo tomo #{command_name}")}
+        ERROR
+      end
+
+      private
+
+      def settings_sentence
+        if settings.length == 1
+          return "a value for the #{yellow(settings.first.to_s)} setting."
+        end
+
+        sentence = "values for these settings:\n\n  "
+        sentence << settings.map { |s| yellow(s.to_s) }.join("\n  ")
+      end
+    end
+  end
+end

data/lib/tomo/runtime/task_aborted_error.rb

@@ -0,0 +1,15 @@
+module Tomo
+  class Runtime
+    class TaskAbortedError < Tomo::Error
+      attr_accessor :task, :host
+
+      def to_console
+        <<~ERROR
+          The #{yellow(task)} task failed on #{yellow(host)}.
+
+          #{red(message)}
+        ERROR
+      end
+    end
+  end
+end

data/lib/tomo/runtime/task_runner.rb

@@ -0,0 +1,56 @@
+module Tomo
+  class Runtime
+    class TaskRunner
+      extend Forwardable
+
+      def_delegators :@context, :paths, :settings
+      attr_reader :context
+
+      def initialize(plugins_registry:, settings:)
+        interpolated_settings = SettingsInterpolation.interpolate(
+          plugins_registry.settings.merge(settings)
+        )
+        @helper_modules = plugins_registry.helper_modules.freeze
+        @context = Context.new(interpolated_settings)
+        @tasks_by_name = plugins_registry.bind_tasks(context).freeze
+        freeze
+      end
+
+      def validate_task!(name)
+        return if tasks_by_name.key?(name)
+
+        UnknownTaskError.raise_with(
+          name,
+          unknown_task: name,
+          known_tasks: tasks_by_name.keys
+        )
+      end
+
+      def run(task:, remote:)
+        validate_task!(task)
+        Current.with(task: task, remote: remote) do
+          Tomo.logger.task_start(task)
+          tasks_by_name[task].call
+        end
+      end
+
+      def connect(host)
+        Current.with(host: host) do
+          conn = SSH.connect(host: host, options: SSH::Options.new(settings))
+          remote = Remote.new(conn, context, helper_modules)
+          return remote unless block_given?
+
+          begin
+            return yield(remote)
+          ensure
+            remote&.close if block_given?
+          end
+        end
+      end
+
+      private
+
+      attr_reader :helper_modules, :tasks_by_name
+    end
+  end
+end

data/lib/tomo/runtime/unknown_task_error.rb

@@ -0,0 +1,23 @@
+module Tomo
+  class Runtime
+    class UnknownTaskError < Error
+      attr_accessor :unknown_task, :known_tasks
+
+      def to_console
+        error = <<~ERROR
+          #{yellow(unknown_task)} is not a recognized task.
+          To see a list of all available tasks, run #{blue('tomo tasks')}.
+        ERROR
+
+        # TODO: suggest "did you forget to add the <abc> plugin?"
+
+        sugg = Error::Suggestions.new(
+          dictionary: known_tasks,
+          word: unknown_task
+        )
+        error << sugg.to_console if sugg.any?
+        error
+      end
+    end
+  end
+end

data/lib/tomo/runtime.rb

@@ -0,0 +1,82 @@
+require "time"
+
+module Tomo
+  class Runtime
+    autoload :ConcurrentRubyLoadError, "tomo/runtime/concurrent_ruby_load_error"
+    autoload :ConcurrentRubyThreadPool,
+             "tomo/runtime/concurrent_ruby_thread_pool"
+    autoload :Context, "tomo/runtime/context"
+    autoload :Current, "tomo/runtime/current"
+    autoload :ExecutionPlan, "tomo/runtime/execution_plan"
+    autoload :HostExecutionStep, "tomo/runtime/host_execution_step"
+    autoload :InlineThreadPool, "tomo/runtime/inline_thread_pool"
+    autoload :PrivilegedTask, "tomo/runtime/privileged_task"
+    autoload :SettingsInterpolation, "tomo/runtime/settings_interpolation"
+    autoload :SettingsRequiredError, "tomo/runtime/settings_required_error"
+    autoload :TaskAbortedError, "tomo/runtime/task_aborted_error"
+    autoload :TaskRunner, "tomo/runtime/task_runner"
+    autoload :UnknownTaskError, "tomo/runtime/unknown_task_error"
+
+    attr_reader :tasks
+
+    def initialize(deploy_tasks:, setup_tasks:, hosts:, task_filter:,
+                   settings:, plugins_registry:)
+      @deploy_tasks = deploy_tasks.freeze
+      @setup_tasks = setup_tasks.freeze
+      @hosts = hosts.freeze
+      @task_filter = task_filter.freeze
+      @settings = settings
+      @plugins_registry = plugins_registry
+      @tasks = plugins_registry.task_names
+      freeze
+    end
+
+    def deploy!
+      execution_plan_for(deploy_tasks, release: :new).execute
+    end
+
+    def setup!
+      execution_plan_for(setup_tasks, release: :tmp).execute
+    end
+
+    def run!(task, *args, privileged: false)
+      task = task.dup.extend(PrivilegedTask) if privileged
+      execution_plan_for([task], release: :current, args: args).execute
+    end
+
+    def execution_plan_for(tasks, release: :current, args: [])
+      ExecutionPlan.new(
+        tasks: tasks,
+        hosts: hosts,
+        task_filter: task_filter,
+        task_runner: new_task_runner(release, args)
+      )
+    end
+
+    private
+
+    attr_reader :deploy_tasks, :setup_tasks, :hosts, :task_filter, :settings,
+                :plugins_registry
+
+    def new_task_runner(release_type, args)
+      run_settings = { release_path: release_path_for(release_type) }
+                     .merge(settings)
+                     .merge(run_args: args)
+
+      TaskRunner.new(plugins_registry: plugins_registry, settings: run_settings)
+    end
+
+    def release_path_for(type)
+      start_time = Time.now
+      release = start_time.utc.strftime("%Y%m%d%H%M%S")
+
+      case type
+      when :current then "%<current_path>"
+      when :new then "%<releases_path>/#{release}"
+      when :tmp then "%<tmp_path>/#{release}"
+      else
+        raise ArgumentError, "release: must be :current, :new, or :tmp"
+      end
+    end
+  end
+end

data/lib/tomo/script.rb

@@ -0,0 +1,44 @@
+module Tomo
+  class Script
+    attr_reader :script
+
+    def initialize(script,
+                   echo: true,
+                   pty: false,
+                   raise_on_error: true,
+                   silent: false)
+      @script = script
+      @echo = echo
+      @pty = pty
+      @raise_on_error = raise_on_error
+      @silent = silent
+      freeze
+    end
+
+    def echo?
+      !!@echo
+    end
+
+    def echo_string
+      return nil unless echo?
+
+      @echo == true ? script : @echo
+    end
+
+    def pty?
+      !!@pty
+    end
+
+    def raise_on_error?
+      !!@raise_on_error
+    end
+
+    def silent?
+      !!@silent
+    end
+
+    def to_s
+      script
+    end
+  end
+end

data/lib/tomo/shell_builder.rb

@@ -0,0 +1,108 @@
+require "shellwords"
+
+module Tomo
+  class ShellBuilder
+    def self.raw(string)
+      string.define_singleton_method(:shellescape) { string }
+      string
+    end
+
+    def initialize
+      @env = {}
+      @chdir = []
+      @prefixes = []
+      @umask = nil
+    end
+
+    def chdir(dir)
+      @chdir << dir
+      yield
+    ensure
+      @chdir.pop
+    end
+
+    def env(hash)
+      orig_env = @env
+      @env = orig_env.merge(hash || {})
+      yield
+    ensure
+      @env = orig_env
+    end
+
+    def prepend(*command)
+      prefixes.unshift(*command)
+      yield
+    ensure
+      prefixes.shift(command.length)
+    end
+
+    def umask(mask)
+      orig_umask = @umask
+      @umask = mask
+      yield
+    ensure
+      @umask = orig_umask
+    end
+
+    def build(*command, default_chdir: nil)
+      if @chdir.empty? && default_chdir
+        return chdir(default_chdir) { build(*command) }
+      end
+
+      command_string = command_to_string(*command)
+      modifiers = [cd_chdir, unset_env, export_env, set_umask].compact.flatten
+      [*modifiers, command_string].join(" && ")
+    end
+
+    private
+
+    attr_reader :prefixes
+
+    def command_to_string(*command)
+      command_string = shell_join(*command)
+      return command_string if prefixes.empty?
+
+      "#{shell_join(*prefixes)} #{command_string}"
+    end
+
+    def shell_join(*command)
+      return command.first.to_s if command.length == 1
+
+      command.flatten.compact.map(&:to_s).map(&:shellescape).join(" ")
+    end
+
+    def cd_chdir
+      @chdir.map { |dir| "cd #{dir.to_s.shellescape}" }
+    end
+
+    def unset_env
+      unsets = @env.select { |_, value| value.nil? }
+      return if unsets.empty?
+
+      ["unset", *unsets.map { |entry| entry.first.to_s.shellescape }].join(" ")
+    end
+
+    def export_env
+      exports = @env.reject { |_, value| value.nil? }
+      return if exports.empty?
+
+      [
+        "export",
+        *exports.map do |key, value|
+          "#{key.to_s.shellescape}=#{value.to_s.shellescape}"
+        end
+      ].join(" ")
+    end
+
+    def set_umask
+      return if @umask.nil?
+
+      umask_value = if @umask.is_a?(Integer)
+                      @umask.to_s(8).rjust(4, "0")
+                    else
+                      @umask
+                    end
+      "umask #{umask_value.to_s.shellescape}"
+    end
+  end
+end

data/lib/tomo/ssh/child_process.rb

@@ -0,0 +1,64 @@
+require "open3"
+require "shellwords"
+
+module Tomo
+  module SSH
+    class ChildProcess
+      def self.execute(*command, on_data: ->(data) {})
+        process = new(*command, on_data: on_data)
+        process.wait_for_exit
+        process.result
+      end
+
+      def initialize(*command, on_data:)
+        @command = *command
+        @on_data = on_data
+        @stdout_buffer = StringIO.new
+        @stderr_buffer = StringIO.new
+        Tomo.logger.debug command.map(&:shellescape).join(" ")
+      end
+
+      def wait_for_exit
+        Open3.popen3(*command) do |stdin, stdout, stderr, wait_thread|
+          stdin.close
+          stdout_thread = start_io_thread(stdout, stdout_buffer)
+          stderr_thread = start_io_thread(stderr, stderr_buffer)
+          stdout_thread.join
+          stderr_thread.join
+          @exit_status = wait_thread.value.exitstatus
+        end
+      end
+
+      def result
+        Result.new(
+          exit_status: exit_status,
+          stdout: stdout_buffer.string,
+          stderr: stderr_buffer.string
+        )
+      end
+
+      private
+
+      attr_reader :command, :exit_status, :on_data,
+                  :stdout_buffer, :stderr_buffer
+
+      def start_io_thread(source, buffer)
+        new_thread_inheriting_current_vars do
+          begin
+            while (line = source.gets)
+              on_data&.call(line)
+              buffer << line
+            end
+          rescue IOError # rubocop:disable Lint/HandleExceptions
+          end
+        end
+      end
+
+      def new_thread_inheriting_current_vars(&block)
+        Thread.new(Runtime::Current.variables) do |vars|
+          Runtime::Current.with(vars, &block)
+        end
+      end
+    end
+  end
+end