executo 0.3.12

data/lib/executo/command.rb

@@ -0,0 +1,56 @@
+# frozen_string_literal: true
+
+module Executo
+  class Command
+    include CommandDsl
+    include TaggedLogger
+
+    attr_reader :executo_id, :parameter_values, :status, :stdout, :stderr, :exitstatus
+
+    def initialize(*args)
+      @executo_id = args.first&.delete(:id) || SecureRandom.uuid
+      @errors = ActiveModel::Errors.new(self)
+      @parameter_values = args.first&.delete(:parameter_values) || {}
+      super(*args)
+    end
+
+    def call
+      raise 'missing target' unless target
+
+      perform
+    end
+
+    def process_results(results)
+      state = results[:state]
+      logger.debug("Processing #{state} results")
+      public_send(state.to_sym, results) if respond_to?(state.to_sym)
+    end
+
+    def setup_logger(id)
+      logger_add_tag(self.class.name)
+      logger_add_tag(id)
+    end
+
+    private
+
+    def perform
+      Executo.publish(target: target, command: command, parameters: safe_parameters, feedback: { service: self.class.name, id: executo_id, arguments: attributes.to_h })
+    end
+
+    def safe_parameters
+      local_parameter_values = {}
+      local_parameter_values = implicit_parameter_values if respond_to?(:implicit_parameter_values)
+      local_parameter_values = local_parameter_values.merge(parameter_values)
+
+      parameters.split.map { |parameter| parameter % local_parameter_values }
+    end
+
+    class << self
+      def process_feedback(feedback, results)
+        cmd = new(feedback['arguments'].merge(id: feedback['id']))
+        cmd.setup_logger(feedback['id'])
+        cmd.process_results(results.symbolize_keys)
+      end
+    end
+  end
+end

data/lib/executo/command_dsl.rb

@@ -0,0 +1,36 @@
+# frozen_string_literal: true
+
+module Executo
+  module CommandDsl
+    extend ActiveSupport::Concern
+    include ActiveAttr::Model
+
+    delegate :target, :command, :parameters, :feedback_interval, to: :class
+
+    class_methods do
+      def call(*args)
+        new(*args).call
+      end
+
+      def target(value = nil)
+        @target = value if value.present?
+        @target
+      end
+
+      def command(value = nil)
+        @command = value if value.present?
+        @command
+      end
+
+      def parameters(value = nil)
+        @parameters = value if value.present?
+        @parameters || ''
+      end
+
+      def feedback_interval(value = nil)
+        @feedback_interval = value if value.present?
+        @feedback_interval || 10
+      end
+    end
+  end
+end

data/lib/executo/commands/imapsync_test.rb

@@ -0,0 +1,52 @@
+# frozen_string_literal: true
+
+# Call with:
+# ImapSyncTest.call(mailbox_id: 'abc', parameter_values: {host1: 1, user1: 1, password1: 1, host2: 2, user2: 2, password2: 2})
+
+class ImapSyncTest < Executo::Command
+  target 'localhost'
+  command '/usr/local/bin/imapsync'
+  parameters '--dry --host1 %<host1>s --user1 %<user1>s --password1 %<password1>s --host2 %<host2>s --user2 %<user2>s --password2 %<password2>s --logfile %<logfile>s --delete2'
+  attribute :mailbox_id
+  feedback_interval 10
+
+  # Callback for 'started' messages. results contain stdout/stderr, atribute mailbox_id will already be set.
+  def started(results)
+    logger.info "Started #{mailbox_id}"
+  end
+
+  # Process intermediate output from the command. results contain stdout/stderr/runtime_seconds
+  def output(results)
+    find_current_status(results[:stdout])
+  end
+
+  # Process completed output from the command. results contain stdout/stderr/exitstatus/runtime_seconds
+  def completed(results)
+    binding.pry
+  end
+
+  # Process failed output from the command. results contain stdout/stderr/exitstatus/runtime_seconds
+  def failed(results)
+  end
+
+  # Process final output from the command. results contain runtime_seconds
+  def finished(results)
+  end
+
+  private
+
+  def find_current_status(lines)
+    line = lines.reverse.find { |l| l =~ %r[\d+/\d+ msgs left] }
+    return unless line
+
+    match = line.match(%r[(?<current>\d+)/(?<total>\d+) msgs left])
+
+    logger.info "Processing #{mailbox_id} - #{match[:current]}/#{match[:total]} (#{(match[:current].to_i.to_f / match[:total].to_i).to_i }%)"
+  end
+
+  def implicit_parameter_values
+    {
+      logfile: "/tmp/imapsync_#{executo_id}.log"
+    }
+  end
+end

data/lib/executo/configuration.rb

@@ -0,0 +1,21 @@
+# frozen_string_literal: true
+
+module Executo
+  class Configuration
+    attr_accessor :redis
+    attr_accessor :active_job_redis
+    attr_writer   :logger
+
+    def initialize
+      @redis = {}
+      @active_job_redis = {}
+      @logger = ActiveSupport::TaggedLogging.new(Logger.new($stdout))
+      @logger.level = Logger::INFO
+    end
+
+    # logger [Object].
+    def logger
+      @logger.is_a?(Proc) ? instance_exec(&@logger) : @logger
+    end
+  end
+end

data/lib/executo/encrypted_worker.rb

@@ -0,0 +1,22 @@
+# frozen_string_literal: true
+
+require 'executo/worker'
+
+module Executo
+  class EncryptedWorker < Worker
+    # @param [String] encrypted_command
+    # @param [Array] encrypted_params
+    # @param [Hash] encrypted_options
+    def perform(encrypted_command, encrypted_params = [], encrypted_options = {})
+      Executo.config.logger.debug "encrypted_command: #{encrypted_command}"
+      Executo.config.logger.debug "encrypted_params: #{encrypted_params}"
+      Executo.config.logger.debug "encrypted_options: #{encrypted_options}"
+
+      command = Executo.decrypt(encrypted_command)
+      params = Executo.decrypt(encrypted_params)
+      options = Executo.decrypt(encrypted_options)
+
+      super(command, params, options)
+    end
+  end
+end

data/lib/executo/feedback_process_job.rb

@@ -0,0 +1,15 @@
+# frozen_string_literal: true
+
+module Executo
+  class FeedbackProcessJob < ActiveJob::Base
+    def perform(feedback, results)
+      feedback_service_class = feedback['service']&.safe_constantize
+      unless feedback_service_class
+        Executo.logger.error("Feedback service #{feedback['service']} not found")
+        return
+      end
+
+      feedback_service_class.process_feedback(feedback, results)
+    end
+  end
+end

data/lib/executo/feedback_process_service.rb

@@ -0,0 +1,41 @@
+# frozen_string_literal: true
+
+module Executo
+  class FeedbackProcessService
+    include Executo::TaggedLogger
+
+    attr_reader :id, :state, :exitstatus, :stdout, :stderr
+    attr_writer :arguments
+
+    def initialize(feedback, results)
+      @id = feedback['id']
+      @state = results['state']
+      @exitstatus = results['exitstatus']
+      @stdout = results['stdout'] || []
+      @stderr = results['stderr'] || []
+      @arguments = feedback['arguments'] || {}
+    end
+
+    def call
+      logger_add_tag(self.class.name)
+      logger_add_tag(id)
+      perform
+    end
+
+    private
+
+    def perform; end
+
+    class << self
+      def arguments(*names)
+        names.each do |name|
+          define_method(name) { instance_variable_get('@arguments')[name.to_s] }
+        end
+      end
+
+      def process_feedback(feedback, results)
+        new(feedback, results).call
+      end
+    end
+  end
+end

data/lib/executo/scheduler_worker.rb

@@ -0,0 +1,33 @@
+# frozen_string_literal: true
+
+# https://gist.github.com/jordinl83/08ad9afd8f5046ddd9d38bcebf373e74
+
+module Executo
+  class SchedulerWorker
+    include Sidekiq::Worker
+    sidekiq_options queue: 'critical'
+
+    def perform
+      execution_time = Time.now.utc
+      execution_time -= execution_time.sec
+
+      self.class.perform_at(execution_time + 60) unless scheduled?
+
+      # SCHEDULE.each do |(worker_class, schedule_lambda)|
+      #   worker_class.perform_async if !scheduled?(worker_class) && schedule_lambda.call(execution_time)
+      # end
+    end
+
+    def scheduled?(worker_class = self.class)
+      scheduled_workers[worker_class.name]
+    end
+
+    private
+
+    def scheduled_workers
+      @scheduled_workers ||= Sidekiq::ScheduledSet.new.entries.each_with_object({}) do |item, hash|
+        hash[item['class']] = true
+      end
+    end
+  end
+end

data/lib/executo/tagged_logger.rb

@@ -0,0 +1,19 @@
+# frozen_string_literal: true
+
+module Executo
+  module TaggedLogger
+    extend ActiveSupport::Concern
+
+    included do
+      def logger_add_tag(tag)
+        return unless tag.present?
+
+        @logger = logger.tagged(tag)
+      end
+
+      def logger
+        @logger ||= Executo.logger
+      end
+    end
+  end
+end

data/lib/executo/version.rb

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+module Executo
+  VERSION = '0.3.12'
+end

data/lib/executo/worker.rb

@@ -0,0 +1,102 @@
+# frozen_string_literal: true
+
+module Executo
+  class Worker
+    include Sidekiq::Worker
+    include Executo::TaggedLogger
+
+    attr_reader :options, :output, :flushable_output
+
+    # @param [String] command
+    # @param [Array] params
+    # @param [Hash] options
+    def perform(command, params = [], options = {})
+      @options = options
+      @output ||= Hash.new([])
+      @flushable_output ||= Hash.new([])
+      @started_at = Time.current
+      @flushed_at = Time.current
+
+      logger_add_tag('Worker')
+      logger_add_tag(options.dig('feedback', 'id'))
+      logger_add_tag(command)
+
+      logger.debug "params: #{params}"
+      logger.debug "options: #{options}"
+
+      send_feedback(state: 'started')
+
+      status = execute(command, params, options)
+      send_feedback(
+        state: status.success? ? 'completed' : 'failed',
+        exitstatus: status.exitstatus.to_i,
+        stdout: output[:stdout],
+        stderr: output[:stderr],
+        pid: status.pid
+      )
+    rescue StandardError => e
+      logger.error "Exception: #{e.class} - #{e.message}"
+      logger.error e.backtrace.join("\n")
+      send_feedback(state: 'failed')
+    ensure
+      send_feedback(state: 'finished')
+    end
+
+    private
+
+    def execute(command, params = [], options = {})
+      argument_list = [command] + params
+      stdin_content = options['stdin_content'] || []
+      stdin_newlines = options.key?('stdin_newlines') ? options['stdin_newlines'] : true
+      shell_escape = options.key?('shell_escape') ? options['shell_escape'] : true
+
+      CLI.run(argument_list, stdout: ->(line) { register_output(:stdout, line) }, stderr: ->(line) { register_output(:stderr, line) }, stdin_content: stdin_content, stdin_newlines: stdin_newlines, shell_escape: shell_escape)
+    end
+
+    def register_output(channel, value)
+      return unless value
+
+      value = value.split("\n").map { |line| line.chomp.force_encoding('utf-8') }
+      output[channel] += value
+      flushable_output[channel] += value
+      logger.debug "Output: #{channel}: #{value}"
+      flush_if_needed
+    end
+
+    def flush_if_needed
+      return if time_since_flushed < (options.dig('feedback', 'flush_interval') || 10)
+      return if flushable_output[:stdout].blank? && flushable_output[:stderr].blank?
+
+      flush_stdout = flushable_output[:stdout].dup
+      flushable_output[:stdout].clear
+      flush_stderr = flushable_output[:stderr].dup
+      flushable_output[:stderr].clear
+
+      send_feedback(state: 'output', stdout: flush_stdout, stderr: flush_stderr)
+      @flushed_at = Time.current
+    end
+
+    def time_since_flushed
+      (Time.current - @flushed_at).to_i
+    end
+
+    def send_feedback(results)
+      results.merge!(runtime_seconds: (Time.current - @started_at).to_i)
+      logger.info "Command #{results[:state]} after #{results[:runtime_seconds]} seconds"
+      Sidekiq::Client.new(Executo.active_job_connection_pool).push(
+        'class' => ActiveJob::QueueAdapters::SidekiqAdapter::JobWrapper,
+        'queue' => 'default',
+        'wrapped' => 'Executo::FeedbackProcessJob',
+        'args' => [
+          {
+            'job_class' => 'Executo::FeedbackProcessJob',
+            'arguments' => [
+              options['feedback'],
+              results.deep_stringify_keys
+            ]
+          }
+        ]
+      )
+    end
+  end
+end

data/lib/executo.rb

@@ -0,0 +1,106 @@
+# frozen_string_literal: true
+
+require 'active_support/message_encryptor'
+require 'active_support/tagged_logging'
+require 'sidekiq'
+require 'active_job'
+require 'securerandom'
+require 'active_attr'
+
+require 'executo/cli'
+require 'executo/tagged_logger'
+require 'executo/configuration'
+require 'executo/version'
+require 'executo/encrypted_worker'
+require 'executo/scheduler_worker'
+require 'executo/worker'
+require 'executo/feedback_process_job'
+require 'executo/feedback_process_service'
+require 'executo/command_dsl'
+require 'executo/command'
+
+require 'executo/commands/imapsync_test'
+
+module Executo
+  class Error < StandardError; end
+
+  class << self
+    attr_reader :config
+
+    delegate :logger, to: :config
+
+    def setup
+      @config = Configuration.new
+      yield config
+    end
+
+    def cryptor
+      @cryptor ||= ActiveSupport::MessageEncryptor.new(ENV['EXECUTO_KEY'])
+    end
+
+    def encrypt(obj)
+      cryptor.encrypt_and_sign(obj)
+    end
+
+    def decrypt(string)
+      cryptor.decrypt_and_verify(string)
+    end
+
+    ##
+    # Publishes a command to be executed on a target
+    #
+    # @param [String] target a server or a role
+    # @param [String] command command to be executed
+    # @param [Array] params params for the command
+    # @param [Boolean] encrypt whether to encrypt all parameters
+    # @param [Hash] options options for the worker
+    # @param [Hash] job_options, options for sidekiq, valid options are:
+    #        queue - the named queue to use, default 'default'
+    #        class - the worker class to call, required
+    #        args - an array of simple arguments to the perform method, must be JSON-serializable
+    #        at - timestamp to schedule the job (optional), must be Numeric (e.g. Time.now.to_f)
+    #        retry - whether to retry this job if it fails, default true or an integer number of retries
+    #        backtrace - whether to save any error backtrace, default false
+    def publish(target:, command:, parameters: [], encrypt: false, options: {}, job_options: {}, feedback: {})
+      options['feedback'] = feedback&.stringify_keys
+      options['feedback']['id'] ||= SecureRandom.uuid
+
+      args = [command, parameters, options.deep_stringify_keys]
+      args = args.map { |a| encrypt(a) } if encrypt
+
+      sidekiq_options = { 'retry' => 0 }.merge(job_options).merge(
+        'queue' => target,
+        'class' => encrypt ? 'Executo::EncryptedWorker' : 'Executo::Worker',
+        'args' => args
+      )
+
+      if defined?(Rails) && Rails.env.test?
+        $executo_jobs ||= {}
+        $executo_jobs[options.dig('feedback', 'id')] = sidekiq_options
+      else
+        Sidekiq::Client.new(connection_pool).push(sidekiq_options)
+      end
+
+      logger.info("Published #{command} to #{target} with id #{options['feedback']['id']}")
+      options.dig('feedback', 'id')
+    end
+
+    def schedule(target, list)
+      options = {
+        'retry' => 0,
+        'queue' => target,
+        'class' => 'Executo::SetScheduleWorker',
+        'args' => list
+      }
+      Sidekiq::Client.new(connection_pool).push(options)
+    end
+
+    def connection_pool
+      @connection_pool ||= ConnectionPool.new(size: 5, timeout: 5) { Redis.new(config.redis) }
+    end
+
+    def active_job_connection_pool
+      @active_job_connection_pool ||= ConnectionPool.new(size: 5, timeout: 5) { Redis.new(config.active_job_redis) }
+    end
+  end
+end

data/sidekiq.yml

@@ -0,0 +1,16 @@
+# Sample configuration file for Sidekiq.
+# Options here can still be overridden by cmd line args.
+# Place this file at config/sidekiq.yml and Sidekiq will
+# pick it up automatically.
+---
+:verbose: false
+:concurrency: 2
+
+# Set timeout to 8 on Heroku, longer if you manage your own systems.
+:timeout: 30
+
+# Sidekiq will run this file through ERB when reading it so you can
+# even put in dynamic logic, like a host-specific queue.
+# http://www.mikeperham.com/2013/11/13/advanced-sidekiq-host-specific-queues/
+:queues:
+  - localhost

data/support/Gemfile

@@ -0,0 +1,8 @@
+# frozen_string_literal: true
+
+source 'https://rubygems.org'
+
+git_source(:github) { |repo_name| "https://github.com/#{repo_name}" }
+git_source(:entdec) { |repo_name| "git@code.entropydecelerator.com:#{repo_name}.git" }
+
+gem 'executo', entdec: 'components/executo', branch: :master

data/support/boot.rb

@@ -0,0 +1,41 @@
+# frozen_string_literal: true
+
+require 'executo'
+require 'net/http'
+require 'uri'
+
+Executo.setup do |config|
+  config.redis = { url: 'redis://localhost:6379/1' }
+  config.callback = lambda do |state, exitstatus, stdout, stderr, context|
+    callback_url = context['options'].delete('callback_url')
+    break unless callback_url
+
+    data = {
+      state: state,
+      command: context['command'],
+      params: context['params'],
+      options: context['options']
+    }
+
+    data[:exitstatus] = exitstatus.to_i if exitstatus
+    data[:stdout] = stdout.force_encoding('utf-8') if stdout
+    data[:stderr] = stderr.force_encoding('utf-8') if stderr
+    data[:pid] = context['pid'].to_i if context['pid']
+
+    uri = URI.parse(callback_url)
+    http = Net::HTTP.new(uri.host, uri.port)
+    http.use_ssl = uri.scheme == 'https'
+    http.verify_mode = OpenSSL::SSL::VERIFY_NONE
+    http.start do
+      http.request_post(
+        uri.path,
+        data.to_json,
+        'Content-Type': 'application/json'
+      )
+    end
+  end
+end
+
+Sidekiq.configure_server do |config|
+  config.redis = Executo.config.redis
+end

data/support/executo.conf

@@ -0,0 +1,21 @@
+description "Executo"
+
+start on runlevel [2345]
+stop on runlevel [06]
+
+env MALLOC_ARENA_MAX=2
+
+respawn
+respawn limit 5 10
+
+normal exit 0 TERM
+reload signal TSTP
+
+script
+exec /bin/bash <<'EOT'
+  # Logs to /var/log/upstart/executo.log
+  cd /data/executo/
+  exec bundle exec sidekiq -r /data/executo/boot.rb -C /data/executo/sidekiq.yml
+EOT
+
+end script

data/support/executo.service

@@ -0,0 +1,26 @@
+[Unit]
+Description=executo
+Wants=redis-server.service
+After=multi-user.target redis-server.service
+
+[Service]
+Type=simple
+WorkingDirectory=/data/executo
+ExecStart=/usr/local/bin/bundle exec sidekiq -r /data/executo/boot.rb -C /var/apps/our-project/config/sidekiq.yml
+User=pi
+Group=pi
+
+# if we crash, restart
+RestartSec=1
+Restart=always
+
+# Logs go to syslog
+# View with "sudo journalctl -u executo"
+StandardOutput=syslog
+StandardError=syslog
+
+# This will default to "bundler" if we don't specify it
+SyslogIdentifier=executo
+
+[Install]
+WantedBy=multi-user.target

data/support/sidekiq.yml

@@ -0,0 +1,17 @@
+# Sample configuration file for Sidekiq.
+# Options here can still be overridden by cmd line args.
+# Place this file at config/sidekiq.yml and Sidekiq will
+# pick it up automatically.
+---
+:verbose: true
+:concurrency: 2
+
+# Set timeout to 8 on Heroku, longer if you manage your own systems.
+:timeout: 30
+
+# Sidekiq will run this file through ERB when reading it so you can
+# even put in dynamic logic, like a host-specific queue.
+# http://www.mikeperham.com/2013/11/13/advanced-sidekiq-host-specific-queues/
+:queues:
+  - g02sj01
+

data/test.rb

@@ -0,0 +1,20 @@
+# frozen_string_literal: true
+
+require 'bundler/setup'
+require 'executo'
+require 'active_support/core_ext/hash'
+require 'pry'
+
+require 'sidekiq'
+require 'active_job'
+
+ActiveJob::Base.queue_adapter = :sidekiq
+
+Executo.setup do |config|
+  config.redis = { url: 'redis://localhost:6379/1' }
+  config.active_job_redis = { url: 'redis://localhost:6379/0' }
+end
+
+Executo.publish('localhost', 'ls', ['-al'], feedback: { service: 'LsProcessService', args: { now: Time.now } })
+
+puts 'Done'

data/uptime.rb

@@ -0,0 +1,4 @@
+require 'curb'
+
+http = Curl.get("http://www.google.com/")
+puts http.body_str