cronicle 0.1.0

data/lib/cronicle/client.rb

@@ -0,0 +1,161 @@
+class Cronicle::Client
+  include Cronicle::Logger::Helper
+
+  DEFAULTS = {
+    :concurrency => 10,
+    :libexec => '/var/lib/cronicle/libexec'
+  }
+
+  def initialize(host_list, options = {})
+    @host_list = host_list
+    @options = DEFAULTS.merge(options)
+  end
+
+  def apply(file)
+    walk(file)
+  end
+
+  def exec(file, name)
+    name = name.to_s
+    jobs = load_file(file)
+    jobs_by_host = select_host(jobs, name)
+
+    if jobs_by_host.empty?
+      raise "Definition cannot be found: Job `#{name}`"
+    end
+
+    parallel_each(jobs_by_host) do |host, jobs_by_user|
+      run_driver(host) do |driver|
+        jobs_by_user.each do |user, jobs|
+          driver.execute_job(user, jobs)
+        end
+      end
+    end
+  end
+
+  private
+
+  def walk(file)
+    jobs = load_file(file)
+    jobs_by_host = select_host(jobs)
+    exported = export_cron(jobs_by_host.keys)
+    walk_hosts(jobs_by_host, exported)
+  end
+
+  def export_cron(host_list)
+    driver = Cronicle::Driver.new(host_list, @options)
+    Cronicle::Exporter.export(driver, @options)
+  end
+
+  def walk_hosts(jobs_by_host, exported)
+    parallel_each(jobs_by_host) do |host, jobs_by_user|
+      exported_by_user = exported.delete(host) || {}
+      walk_host(host, jobs_by_user, exported_by_user)
+    end
+
+    parallel_each(exported) do |host, exported_by_user|
+      run_driver(host) do |driver|
+        scripts_by_user = {}
+
+        exported_by_user.each do |user, scripts|
+          scripts_by_user[user] = scripts
+        end
+
+        driver.delete_job(scripts_by_user)
+      end
+    end
+  end
+
+  def walk_host(host, jobs_by_user, exported_by_user)
+    run_driver(host) do |driver|
+      jobs_by_user.each do |user, jobs|
+        scripts = exported_by_user.delete(user) || {}
+        walk_jobs(driver, user, jobs, scripts)
+      end
+
+      exported_by_user.each do |user, scripts|
+        driver.delete_job(user => scripts)
+      end
+    end
+  end
+
+  def walk_jobs(driver, user, jobs, cron_cmds)
+    jobs.each do |name, job|
+      next unless job[:schedule]
+      current_cmd = cron_cmds.delete(name)
+
+      if current_cmd
+        driver.update_job(user, name, job, current_cmd)
+      else
+        driver.create_job(user, name, job)
+      end
+    end
+
+    cron_cmds.each do |name, current_cmd|
+      driver.delete_job(user => {name => current_cmd})
+    end
+  end
+
+  def run_driver(host)
+    driver = Cronicle::Driver.new(Array(host), @options)
+
+    if driver.test_sudo
+      yield(driver)
+    end
+  end
+
+  def select_host(jobs, target_name = nil)
+    hosts = Hash.new do |jobs_by_host, host|
+      jobs_by_host[host] = Hash.new do |jobs_by_user, user|
+        jobs_by_user[user] = {}
+      end
+    end
+
+    jobs.each do |job|
+      job_hash = job[:job]
+      job_user = job_hash[:user]
+      job_name = job_hash[:name]
+      servers = job[:servers]
+
+      if target_name and job_name != target_name
+        next
+      end
+
+      selected_hots = @host_list.select(
+        :servers => servers,
+        :roles => job[:roles]
+      )
+
+      # Add hosts that is defined in DSL
+      dsl_hosts = servers.select {|srvr|
+        srvr.kind_of?(String) or srvr.kind_of?(Symbol)
+      }.map(&:to_s)
+
+      (selected_hots + dsl_hosts).uniq.each do |h|
+        if hosts[h][job_user][job_name]
+          log(:warn, "Job is duplicated", :color => :yellow, :host => h, :user => job_user, :job => job_name)
+        else
+          hosts[h][job_user][job_name] = job_hash
+        end
+      end
+    end
+
+    hosts
+  end
+
+  def load_file(file)
+    if file.kind_of?(String)
+      open(file) do |f|
+        Cronicle::DSL.parse(f.read, file, @options)
+      end
+    elsif [File, Tempfile].any? {|i| file.kind_of?(i) }
+      Cronicle::DSL.parse(file.read, file.path, @options)
+    else
+      raise TypeError, "Cannot convert #{file} into File"
+    end
+  end
+
+  def parallel_each(enum, &block)
+    Parallel.each(enum, :in_threads => @options[:concurrency], &block)
+  end
+end

data/lib/cronicle/driver.rb

@@ -0,0 +1,157 @@
+class Cronicle::Driver
+  CRON_DIRS = %w(/var/spool/cron/crontabs /var/spool/cron)
+
+  attr_reader :hosts
+
+  def initialize(hosts, options = nil)
+    @hosts = hosts
+    @options = options
+  end
+
+  def execute(&block)
+    coordinator = SSHKit::Coordinator.new(@hosts)
+    hosts = coordinator.hosts
+
+    hosts.each do |host|
+      host.instance_variable_set(:@options, @options)
+    end
+
+    runner_opts = @options[:runner_options] || {}
+    runner = SSHKit::Runner::Group.new(hosts, runner_opts, &block)
+    runner.group_size = @options[:concurrency]
+    runner.execute
+  end
+
+  def export_crontab
+    crontabs_by_host = {}
+    libexec_by_host = {}
+
+    execute do
+      crontabs_by_host[host.hostname] = fetch_crontabs
+      libexec_by_host[host.hostname] = fetch_libexec_scripts
+    end
+
+    [crontabs_by_host, libexec_by_host]
+  end
+
+  def execute_job(user, jobs)
+    execute do
+      mktemp do |temp_dir|
+        jobs.each do |name, job|
+          log_opts = {:color => :cyan, :host => host.hostname, :user => user, :job => name}
+          log_msg = 'Execute job'
+
+          if host.options[:dry_run]
+            if host.options[:verbose]
+              content = job[:content].each_line.map {|l| ' ' + l }.join
+              log_msg << "\n" << content.chomp << "\n"
+            end
+
+            log_for_cronicle(:info, log_msg, log_opts)
+            next
+          else
+            log_for_cronicle(:info, log_msg, log_opts)
+          end
+
+          upload_script(temp_dir, name, job[:content]) do |temp_script|
+            command = sudo(:_execute, temp_script, :user => user, :raise_on_non_zero_exit => false)
+            out = command.full_stdout
+            Cronicle::Utils.remove_prompt!(out)
+            host_user_job = {:host => host.hostname, :user => user, :job => name}
+
+            put_log = proc do |level, opts|
+              opts ||= {}
+
+              out.each_line do |line|
+                log_for_cronicle(:info, line.strip, opts.merge(host_user_job))
+              end
+            end
+
+            if command.exit_status.zero?
+              put_log.call(:info)
+            else
+              put_log.call(:error, :color => :red)
+            end
+          end
+        end
+      end
+    end
+  end
+
+  def create_job(user, name, job)
+    create_or_update_job(user, name, job)
+  end
+
+  def update_job(user, name, job, script)
+    job_content = job[:content].chomp
+    script_content = script[:content].chomp
+
+    if [:schedule, :content].any? {|k| job[k].chomp != script[k].chomp }
+      create_or_update_job(user, name, job, script)
+    end
+  end
+
+  def create_or_update_job(user, name, job, script = nil)
+    execute do
+      log_opts = {:host => host.hostname, :user => user, :job => name}
+      log_opts[:color] = script ? :green : :cyan
+      log_msg = (script ? 'Update' : 'Create') + " job: schedule=#{job[:schedule].inspect}"
+
+      if host.options[:verbose]
+        content_orig = script ? script[:content] : ''
+        log_msg << "\n" << Cronicle::Utils.diff(content_orig, job[:content])
+      end
+
+      log_for_cronicle(:info, log_msg, log_opts)
+
+      unless host.options[:dry_run]
+        mktemp(user) do |temp_dir, user_temp_dir|
+          libexec_script = script_path(user, name)
+
+          upload_script(temp_dir, name, job[:content]) do |temp_script|
+            temp_entry = temp_script + '.entry'
+            sudo(:execute, :mkdir, '-p', user_libexec_dir(user))
+            sudo(:execute, :cp, temp_script, libexec_script)
+            sudo(:execute, :touch, user_crontab(user))
+            delete_cron_entry(user, name)
+            add_cron_entry(user, name, job[:schedule], user_temp_dir)
+          end
+        end
+      end
+    end
+  end
+
+  def delete_job(scripts_by_user, target_name = nil)
+    execute do
+      scripts_by_user.each do |user, scripts|
+        scripts.each do |name, script|
+          next if target_name && target_name != name
+
+          log_opts = {:color => :red, :host => host.hostname, :user => user, :job => name}
+          log_msg = "Delete job: schedule=#{script[:schedule].inspect}"
+          log_msg << "\n" << Cronicle::Utils.diff(script[:content], '') if host.options[:verbose]
+          log_for_cronicle(:info, log_msg, log_opts)
+
+          unless host.options[:dry_run]
+            delete_cron_entry(user, name)
+            sudo(:execute, :rm, '-f', script[:path], :raise_on_non_zero_exit => false)
+          end
+        end
+      end
+    end
+  end
+
+  def test_sudo
+    ok = false
+
+    execute do
+      ok = sudo(:execute, :echo, :raise_on_non_zero_exit => false)
+
+      unless ok
+        log_for_cronicle(:error, 'incorrect sudo password', :color => :red, :host => host.hostname)
+      end
+    end
+
+    ok
+  end
+end

data/lib/cronicle/dsl.rb

@@ -0,0 +1,7 @@
+class Cronicle::DSL
+  class << self
+    def parse(dsl, path, opts = {})
+      Cronicle::DSL::Context.eval(dsl, path, opts).result
+    end
+  end # of class methods
+end

data/lib/cronicle/dsl/context.rb

@@ -0,0 +1,48 @@
+class Cronicle::DSL::Context
+  class << self
+    def eval(dsl, path, opts = {})
+      self.new(path, opts) {
+        Kernel.eval(dsl, binding, path)
+      }
+    end
+  end # of class methods
+
+  attr_reader :result
+
+  def initialize(path, options = {}, &block)
+    @path = path
+    @options = options
+    @result = []
+    instance_eval(&block)
+  end
+
+  def require(file)
+    cronfile = (file =~ %r|\A/|) ? file : File.expand_path(File.join(File.dirname(@path), file))
+
+    if File.exist?(cronfile)
+      instance_eval(File.read(cronfile), cronfile)
+    elsif File.exist?(cronfile + '.rb')
+      instance_eval(File.read(cronfile + '.rb'), cronfile + '.rb')
+    else
+      Kernel.require(file)
+    end
+  end
+
+  def on(target, &block)
+    unless block
+      raise ArgumentError, "Block is required for `on` method"
+    end
+
+    unless target.kind_of?(Hash)
+      raise TypeError, "wrong argument type #{target.class} (expected Hash)"
+    end
+
+    if target.empty?
+      raise ArgumentError, ':servers or :roles is not passed to `on` method'
+    end
+
+    target.assert_valid_keys(:servers, :roles)
+
+    @result.concat(Cronicle::DSL::Context::Job.new(target, &block).result.values)
+  end
+end

data/lib/cronicle/dsl/context/job.rb

@@ -0,0 +1,58 @@
+class Cronicle::DSL::Context::Job
+  def initialize(target, &block)
+    @result = Hash.new {|hash, key|
+      hash[key] = {
+        :servers => Array(target[:servers]),
+        :roles => Array(target[:roles]),
+        :job => {}
+      }
+    }
+
+    instance_eval(&block)
+  end
+
+  attr_reader :result
+
+  def job(name, opts = {}, &block)
+    name = name.to_s
+
+    raise ArgumentError, %!Job name is required! if (name || '').strip.empty?
+
+    if @result.has_key?(name)
+      raise "Job `#{name}`: already defined"
+    end
+
+    unless opts.kind_of?(Hash)
+      raise TypeError, "Job `#{name}`: wrong argument type #{opts.class} (expected Hash)"
+    end
+
+    unless opts[:user]
+      raise ArgumentError, "Job `#{name}`: :user is required"
+    end
+
+    opts.assert_valid_keys(:schedule, :user, :content)
+
+    if opts[:content] and block
+      raise ArgumentError, 'Can not pass :content and block to `job` method'
+    elsif not opts[:content] and not block
+      raise ArgumentError, "Job `#{name}`: :context or block is required"
+    end
+
+    job_hash = @result[name][:job]
+    job_hash[:name] = name
+    job_hash[:user] = opts.fetch(:user).to_s
+    job_hash[:schedule] = opts[:schedule].to_s if opts[:schedule]
+
+    if block
+      source = block.to_raw_source(:strip_enclosure => true).each_line.to_a
+      source = source.shift + source.join.undent
+
+      job_hash[:content] = <<-RUBY
+#!/usr/bin/env ruby
+#{source}
+      RUBY
+    else
+      job_hash[:content] = opts[:content].to_s.undent
+    end
+  end
+end

data/lib/cronicle/exporter.rb

@@ -0,0 +1,56 @@
+class Cronicle::Exporter
+  class << self
+    def export(driver, opts = {})
+      self.new(driver, opts).export
+    end
+  end # of class methods
+
+  def initialize(driver, options = {})
+    @driver = driver
+    @options = options
+  end
+
+  def export
+    crontabs_by_host, libexec_by_host = @driver.export_crontab
+    parse(crontabs_by_host, libexec_by_host)
+  end
+
+  private
+
+  def parse(crontabs_by_host, libexec_by_host)
+    crontabs_by_host.each do |host, crontab_by_user|
+      libexec_contents = libexec_by_host[host] || {}
+
+      crontab_by_user.keys.each do |user|
+        crontab = crontab_by_user[user]
+        crontab_by_user[user] = parse_crontab(crontab, libexec_contents)
+      end
+    end
+
+    crontabs_by_host
+  end
+
+  def parse_crontab(crontab, libexec_contents)
+    scripts = {}
+    libexec_dir = @options.fetch(:libexec)
+
+    crontab.each_line.map(&:strip).each do |line|
+      next if line =~ /\A#/
+
+      md = line.match(/\A(@\w+|\S+(?:\s+\S+){4})\s+(.\S+)(.*)\z/)
+      schedule, path, extra = md.captures if md
+
+      if %r|\A#{Regexp.escape(libexec_dir)}/(?:[^/]+)/(.+)| =~ path
+        name = $1
+
+        scripts[name] = {
+          :schedule => schedule,
+          :path => path,
+          :content => libexec_contents[path]
+        }
+      end
+    end
+
+    scripts
+  end
+end