cronicle 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/.gitignore +19 -0
- data/.rspec +2 -0
- data/.travis.yml +22 -0
- data/Gemfile +4 -0
- data/LICENSE.txt +22 -0
- data/README.md +127 -0
- data/Rakefile +5 -0
- data/Vagrantfile +56 -0
- data/bin/cronicle +6 -0
- data/cronicle.gemspec +31 -0
- data/cronicle.us-east-1.pem.enc +0 -0
- data/lib/cronicle.rb +28 -0
- data/lib/cronicle/cli.rb +105 -0
- data/lib/cronicle/client.rb +161 -0
- data/lib/cronicle/driver.rb +157 -0
- data/lib/cronicle/dsl.rb +7 -0
- data/lib/cronicle/dsl/context.rb +48 -0
- data/lib/cronicle/dsl/context/job.rb +58 -0
- data/lib/cronicle/exporter.rb +56 -0
- data/lib/cronicle/ext/hash_ext.rb +8 -0
- data/lib/cronicle/ext/sshkit_ext.rb +147 -0
- data/lib/cronicle/ext/string_ext.rb +11 -0
- data/lib/cronicle/host_list.rb +89 -0
- data/lib/cronicle/logger.rb +92 -0
- data/lib/cronicle/utils.rb +43 -0
- data/lib/cronicle/version.rb +3 -0
- data/spec/cronicle_create_spec.rb +295 -0
- data/spec/cronicle_delete_spec.rb +334 -0
- data/spec/cronicle_exec_spec.rb +159 -0
- data/spec/cronicle_update_spec.rb +365 -0
- data/spec/host_list_spec.rb +162 -0
- data/spec/spec_helper.rb +150 -0
- metadata +223 -0
@@ -0,0 +1,161 @@
|
|
1
|
+
class Cronicle::Client
|
2
|
+
include Cronicle::Logger::Helper
|
3
|
+
|
4
|
+
DEFAULTS = {
|
5
|
+
:concurrency => 10,
|
6
|
+
:libexec => '/var/lib/cronicle/libexec'
|
7
|
+
}
|
8
|
+
|
9
|
+
def initialize(host_list, options = {})
|
10
|
+
@host_list = host_list
|
11
|
+
@options = DEFAULTS.merge(options)
|
12
|
+
end
|
13
|
+
|
14
|
+
def apply(file)
|
15
|
+
walk(file)
|
16
|
+
end
|
17
|
+
|
18
|
+
def exec(file, name)
|
19
|
+
name = name.to_s
|
20
|
+
jobs = load_file(file)
|
21
|
+
jobs_by_host = select_host(jobs, name)
|
22
|
+
|
23
|
+
if jobs_by_host.empty?
|
24
|
+
raise "Definition cannot be found: Job `#{name}`"
|
25
|
+
end
|
26
|
+
|
27
|
+
parallel_each(jobs_by_host) do |host, jobs_by_user|
|
28
|
+
run_driver(host) do |driver|
|
29
|
+
jobs_by_user.each do |user, jobs|
|
30
|
+
driver.execute_job(user, jobs)
|
31
|
+
end
|
32
|
+
end
|
33
|
+
end
|
34
|
+
end
|
35
|
+
|
36
|
+
private
|
37
|
+
|
38
|
+
def walk(file)
|
39
|
+
jobs = load_file(file)
|
40
|
+
jobs_by_host = select_host(jobs)
|
41
|
+
exported = export_cron(jobs_by_host.keys)
|
42
|
+
walk_hosts(jobs_by_host, exported)
|
43
|
+
end
|
44
|
+
|
45
|
+
def export_cron(host_list)
|
46
|
+
driver = Cronicle::Driver.new(host_list, @options)
|
47
|
+
Cronicle::Exporter.export(driver, @options)
|
48
|
+
end
|
49
|
+
|
50
|
+
def walk_hosts(jobs_by_host, exported)
|
51
|
+
parallel_each(jobs_by_host) do |host, jobs_by_user|
|
52
|
+
exported_by_user = exported.delete(host) || {}
|
53
|
+
walk_host(host, jobs_by_user, exported_by_user)
|
54
|
+
end
|
55
|
+
|
56
|
+
parallel_each(exported) do |host, exported_by_user|
|
57
|
+
run_driver(host) do |driver|
|
58
|
+
scripts_by_user = {}
|
59
|
+
|
60
|
+
exported_by_user.each do |user, scripts|
|
61
|
+
scripts_by_user[user] = scripts
|
62
|
+
end
|
63
|
+
|
64
|
+
driver.delete_job(scripts_by_user)
|
65
|
+
end
|
66
|
+
end
|
67
|
+
end
|
68
|
+
|
69
|
+
def walk_host(host, jobs_by_user, exported_by_user)
|
70
|
+
run_driver(host) do |driver|
|
71
|
+
jobs_by_user.each do |user, jobs|
|
72
|
+
scripts = exported_by_user.delete(user) || {}
|
73
|
+
walk_jobs(driver, user, jobs, scripts)
|
74
|
+
end
|
75
|
+
|
76
|
+
exported_by_user.each do |user, scripts|
|
77
|
+
driver.delete_job(user => scripts)
|
78
|
+
end
|
79
|
+
end
|
80
|
+
end
|
81
|
+
|
82
|
+
def walk_jobs(driver, user, jobs, cron_cmds)
|
83
|
+
jobs.each do |name, job|
|
84
|
+
next unless job[:schedule]
|
85
|
+
current_cmd = cron_cmds.delete(name)
|
86
|
+
|
87
|
+
if current_cmd
|
88
|
+
driver.update_job(user, name, job, current_cmd)
|
89
|
+
else
|
90
|
+
driver.create_job(user, name, job)
|
91
|
+
end
|
92
|
+
end
|
93
|
+
|
94
|
+
cron_cmds.each do |name, current_cmd|
|
95
|
+
driver.delete_job(user => {name => current_cmd})
|
96
|
+
end
|
97
|
+
end
|
98
|
+
|
99
|
+
def run_driver(host)
|
100
|
+
driver = Cronicle::Driver.new(Array(host), @options)
|
101
|
+
|
102
|
+
if driver.test_sudo
|
103
|
+
yield(driver)
|
104
|
+
end
|
105
|
+
end
|
106
|
+
|
107
|
+
def select_host(jobs, target_name = nil)
|
108
|
+
hosts = Hash.new do |jobs_by_host, host|
|
109
|
+
jobs_by_host[host] = Hash.new do |jobs_by_user, user|
|
110
|
+
jobs_by_user[user] = {}
|
111
|
+
end
|
112
|
+
end
|
113
|
+
|
114
|
+
jobs.each do |job|
|
115
|
+
job_hash = job[:job]
|
116
|
+
job_user = job_hash[:user]
|
117
|
+
job_name = job_hash[:name]
|
118
|
+
servers = job[:servers]
|
119
|
+
|
120
|
+
if target_name and job_name != target_name
|
121
|
+
next
|
122
|
+
end
|
123
|
+
|
124
|
+
selected_hots = @host_list.select(
|
125
|
+
:servers => servers,
|
126
|
+
:roles => job[:roles]
|
127
|
+
)
|
128
|
+
|
129
|
+
# Add hosts that is defined in DSL
|
130
|
+
dsl_hosts = servers.select {|srvr|
|
131
|
+
srvr.kind_of?(String) or srvr.kind_of?(Symbol)
|
132
|
+
}.map(&:to_s)
|
133
|
+
|
134
|
+
(selected_hots + dsl_hosts).uniq.each do |h|
|
135
|
+
if hosts[h][job_user][job_name]
|
136
|
+
log(:warn, "Job is duplicated", :color => :yellow, :host => h, :user => job_user, :job => job_name)
|
137
|
+
else
|
138
|
+
hosts[h][job_user][job_name] = job_hash
|
139
|
+
end
|
140
|
+
end
|
141
|
+
end
|
142
|
+
|
143
|
+
hosts
|
144
|
+
end
|
145
|
+
|
146
|
+
def load_file(file)
|
147
|
+
if file.kind_of?(String)
|
148
|
+
open(file) do |f|
|
149
|
+
Cronicle::DSL.parse(f.read, file, @options)
|
150
|
+
end
|
151
|
+
elsif [File, Tempfile].any? {|i| file.kind_of?(i) }
|
152
|
+
Cronicle::DSL.parse(file.read, file.path, @options)
|
153
|
+
else
|
154
|
+
raise TypeError, "Cannot convert #{file} into File"
|
155
|
+
end
|
156
|
+
end
|
157
|
+
|
158
|
+
def parallel_each(enum, &block)
|
159
|
+
Parallel.each(enum, :in_threads => @options[:concurrency], &block)
|
160
|
+
end
|
161
|
+
end
|
@@ -0,0 +1,157 @@
|
|
1
|
+
class Cronicle::Driver
|
2
|
+
CRON_DIRS = %w(/var/spool/cron/crontabs /var/spool/cron)
|
3
|
+
|
4
|
+
attr_reader :hosts
|
5
|
+
|
6
|
+
def initialize(hosts, options = nil)
|
7
|
+
@hosts = hosts
|
8
|
+
@options = options
|
9
|
+
end
|
10
|
+
|
11
|
+
def execute(&block)
|
12
|
+
coordinator = SSHKit::Coordinator.new(@hosts)
|
13
|
+
hosts = coordinator.hosts
|
14
|
+
|
15
|
+
hosts.each do |host|
|
16
|
+
host.instance_variable_set(:@options, @options)
|
17
|
+
end
|
18
|
+
|
19
|
+
runner_opts = @options[:runner_options] || {}
|
20
|
+
runner = SSHKit::Runner::Group.new(hosts, runner_opts, &block)
|
21
|
+
runner.group_size = @options[:concurrency]
|
22
|
+
runner.execute
|
23
|
+
end
|
24
|
+
|
25
|
+
def export_crontab
|
26
|
+
crontabs_by_host = {}
|
27
|
+
libexec_by_host = {}
|
28
|
+
|
29
|
+
execute do
|
30
|
+
crontabs_by_host[host.hostname] = fetch_crontabs
|
31
|
+
libexec_by_host[host.hostname] = fetch_libexec_scripts
|
32
|
+
end
|
33
|
+
|
34
|
+
[crontabs_by_host, libexec_by_host]
|
35
|
+
end
|
36
|
+
|
37
|
+
def execute_job(user, jobs)
|
38
|
+
execute do
|
39
|
+
mktemp do |temp_dir|
|
40
|
+
jobs.each do |name, job|
|
41
|
+
log_opts = {:color => :cyan, :host => host.hostname, :user => user, :job => name}
|
42
|
+
log_msg = 'Execute job'
|
43
|
+
|
44
|
+
if host.options[:dry_run]
|
45
|
+
if host.options[:verbose]
|
46
|
+
content = job[:content].each_line.map {|l| ' ' + l }.join
|
47
|
+
log_msg << "\n" << content.chomp << "\n"
|
48
|
+
end
|
49
|
+
|
50
|
+
log_for_cronicle(:info, log_msg, log_opts)
|
51
|
+
next
|
52
|
+
else
|
53
|
+
log_for_cronicle(:info, log_msg, log_opts)
|
54
|
+
end
|
55
|
+
|
56
|
+
upload_script(temp_dir, name, job[:content]) do |temp_script|
|
57
|
+
command = sudo(:_execute, temp_script, :user => user, :raise_on_non_zero_exit => false)
|
58
|
+
out = command.full_stdout
|
59
|
+
Cronicle::Utils.remove_prompt!(out)
|
60
|
+
host_user_job = {:host => host.hostname, :user => user, :job => name}
|
61
|
+
|
62
|
+
put_log = proc do |level, opts|
|
63
|
+
opts ||= {}
|
64
|
+
|
65
|
+
out.each_line do |line|
|
66
|
+
log_for_cronicle(:info, line.strip, opts.merge(host_user_job))
|
67
|
+
end
|
68
|
+
end
|
69
|
+
|
70
|
+
if command.exit_status.zero?
|
71
|
+
put_log.call(:info)
|
72
|
+
else
|
73
|
+
put_log.call(:error, :color => :red)
|
74
|
+
end
|
75
|
+
end
|
76
|
+
end
|
77
|
+
end
|
78
|
+
end
|
79
|
+
end
|
80
|
+
|
81
|
+
def create_job(user, name, job)
|
82
|
+
create_or_update_job(user, name, job)
|
83
|
+
end
|
84
|
+
|
85
|
+
def update_job(user, name, job, script)
|
86
|
+
job_content = job[:content].chomp
|
87
|
+
script_content = script[:content].chomp
|
88
|
+
|
89
|
+
if [:schedule, :content].any? {|k| job[k].chomp != script[k].chomp }
|
90
|
+
create_or_update_job(user, name, job, script)
|
91
|
+
end
|
92
|
+
end
|
93
|
+
|
94
|
+
def create_or_update_job(user, name, job, script = nil)
|
95
|
+
execute do
|
96
|
+
log_opts = {:host => host.hostname, :user => user, :job => name}
|
97
|
+
log_opts[:color] = script ? :green : :cyan
|
98
|
+
log_msg = (script ? 'Update' : 'Create') + " job: schedule=#{job[:schedule].inspect}"
|
99
|
+
|
100
|
+
if host.options[:verbose]
|
101
|
+
content_orig = script ? script[:content] : ''
|
102
|
+
log_msg << "\n" << Cronicle::Utils.diff(content_orig, job[:content])
|
103
|
+
end
|
104
|
+
|
105
|
+
log_for_cronicle(:info, log_msg, log_opts)
|
106
|
+
|
107
|
+
unless host.options[:dry_run]
|
108
|
+
mktemp(user) do |temp_dir, user_temp_dir|
|
109
|
+
libexec_script = script_path(user, name)
|
110
|
+
|
111
|
+
upload_script(temp_dir, name, job[:content]) do |temp_script|
|
112
|
+
temp_entry = temp_script + '.entry'
|
113
|
+
sudo(:execute, :mkdir, '-p', user_libexec_dir(user))
|
114
|
+
sudo(:execute, :cp, temp_script, libexec_script)
|
115
|
+
sudo(:execute, :touch, user_crontab(user))
|
116
|
+
delete_cron_entry(user, name)
|
117
|
+
add_cron_entry(user, name, job[:schedule], user_temp_dir)
|
118
|
+
end
|
119
|
+
end
|
120
|
+
end
|
121
|
+
end
|
122
|
+
end
|
123
|
+
|
124
|
+
def delete_job(scripts_by_user, target_name = nil)
|
125
|
+
execute do
|
126
|
+
scripts_by_user.each do |user, scripts|
|
127
|
+
scripts.each do |name, script|
|
128
|
+
next if target_name && target_name != name
|
129
|
+
|
130
|
+
log_opts = {:color => :red, :host => host.hostname, :user => user, :job => name}
|
131
|
+
log_msg = "Delete job: schedule=#{script[:schedule].inspect}"
|
132
|
+
log_msg << "\n" << Cronicle::Utils.diff(script[:content], '') if host.options[:verbose]
|
133
|
+
log_for_cronicle(:info, log_msg, log_opts)
|
134
|
+
|
135
|
+
unless host.options[:dry_run]
|
136
|
+
delete_cron_entry(user, name)
|
137
|
+
sudo(:execute, :rm, '-f', script[:path], :raise_on_non_zero_exit => false)
|
138
|
+
end
|
139
|
+
end
|
140
|
+
end
|
141
|
+
end
|
142
|
+
end
|
143
|
+
|
144
|
+
def test_sudo
|
145
|
+
ok = false
|
146
|
+
|
147
|
+
execute do
|
148
|
+
ok = sudo(:execute, :echo, :raise_on_non_zero_exit => false)
|
149
|
+
|
150
|
+
unless ok
|
151
|
+
log_for_cronicle(:error, 'incorrect sudo password', :color => :red, :host => host.hostname)
|
152
|
+
end
|
153
|
+
end
|
154
|
+
|
155
|
+
ok
|
156
|
+
end
|
157
|
+
end
|
data/lib/cronicle/dsl.rb
ADDED
@@ -0,0 +1,48 @@
|
|
1
|
+
class Cronicle::DSL::Context
|
2
|
+
class << self
|
3
|
+
def eval(dsl, path, opts = {})
|
4
|
+
self.new(path, opts) {
|
5
|
+
Kernel.eval(dsl, binding, path)
|
6
|
+
}
|
7
|
+
end
|
8
|
+
end # of class methods
|
9
|
+
|
10
|
+
attr_reader :result
|
11
|
+
|
12
|
+
def initialize(path, options = {}, &block)
|
13
|
+
@path = path
|
14
|
+
@options = options
|
15
|
+
@result = []
|
16
|
+
instance_eval(&block)
|
17
|
+
end
|
18
|
+
|
19
|
+
def require(file)
|
20
|
+
cronfile = (file =~ %r|\A/|) ? file : File.expand_path(File.join(File.dirname(@path), file))
|
21
|
+
|
22
|
+
if File.exist?(cronfile)
|
23
|
+
instance_eval(File.read(cronfile), cronfile)
|
24
|
+
elsif File.exist?(cronfile + '.rb')
|
25
|
+
instance_eval(File.read(cronfile + '.rb'), cronfile + '.rb')
|
26
|
+
else
|
27
|
+
Kernel.require(file)
|
28
|
+
end
|
29
|
+
end
|
30
|
+
|
31
|
+
def on(target, &block)
|
32
|
+
unless block
|
33
|
+
raise ArgumentError, "Block is required for `on` method"
|
34
|
+
end
|
35
|
+
|
36
|
+
unless target.kind_of?(Hash)
|
37
|
+
raise TypeError, "wrong argument type #{target.class} (expected Hash)"
|
38
|
+
end
|
39
|
+
|
40
|
+
if target.empty?
|
41
|
+
raise ArgumentError, ':servers or :roles is not passed to `on` method'
|
42
|
+
end
|
43
|
+
|
44
|
+
target.assert_valid_keys(:servers, :roles)
|
45
|
+
|
46
|
+
@result.concat(Cronicle::DSL::Context::Job.new(target, &block).result.values)
|
47
|
+
end
|
48
|
+
end
|
@@ -0,0 +1,58 @@
|
|
1
|
+
class Cronicle::DSL::Context::Job
|
2
|
+
def initialize(target, &block)
|
3
|
+
@result = Hash.new {|hash, key|
|
4
|
+
hash[key] = {
|
5
|
+
:servers => Array(target[:servers]),
|
6
|
+
:roles => Array(target[:roles]),
|
7
|
+
:job => {}
|
8
|
+
}
|
9
|
+
}
|
10
|
+
|
11
|
+
instance_eval(&block)
|
12
|
+
end
|
13
|
+
|
14
|
+
attr_reader :result
|
15
|
+
|
16
|
+
def job(name, opts = {}, &block)
|
17
|
+
name = name.to_s
|
18
|
+
|
19
|
+
raise ArgumentError, %!Job name is required! if (name || '').strip.empty?
|
20
|
+
|
21
|
+
if @result.has_key?(name)
|
22
|
+
raise "Job `#{name}`: already defined"
|
23
|
+
end
|
24
|
+
|
25
|
+
unless opts.kind_of?(Hash)
|
26
|
+
raise TypeError, "Job `#{name}`: wrong argument type #{opts.class} (expected Hash)"
|
27
|
+
end
|
28
|
+
|
29
|
+
unless opts[:user]
|
30
|
+
raise ArgumentError, "Job `#{name}`: :user is required"
|
31
|
+
end
|
32
|
+
|
33
|
+
opts.assert_valid_keys(:schedule, :user, :content)
|
34
|
+
|
35
|
+
if opts[:content] and block
|
36
|
+
raise ArgumentError, 'Can not pass :content and block to `job` method'
|
37
|
+
elsif not opts[:content] and not block
|
38
|
+
raise ArgumentError, "Job `#{name}`: :context or block is required"
|
39
|
+
end
|
40
|
+
|
41
|
+
job_hash = @result[name][:job]
|
42
|
+
job_hash[:name] = name
|
43
|
+
job_hash[:user] = opts.fetch(:user).to_s
|
44
|
+
job_hash[:schedule] = opts[:schedule].to_s if opts[:schedule]
|
45
|
+
|
46
|
+
if block
|
47
|
+
source = block.to_raw_source(:strip_enclosure => true).each_line.to_a
|
48
|
+
source = source.shift + source.join.undent
|
49
|
+
|
50
|
+
job_hash[:content] = <<-RUBY
|
51
|
+
#!/usr/bin/env ruby
|
52
|
+
#{source}
|
53
|
+
RUBY
|
54
|
+
else
|
55
|
+
job_hash[:content] = opts[:content].to_s.undent
|
56
|
+
end
|
57
|
+
end
|
58
|
+
end
|
@@ -0,0 +1,56 @@
|
|
1
|
+
class Cronicle::Exporter
|
2
|
+
class << self
|
3
|
+
def export(driver, opts = {})
|
4
|
+
self.new(driver, opts).export
|
5
|
+
end
|
6
|
+
end # of class methods
|
7
|
+
|
8
|
+
def initialize(driver, options = {})
|
9
|
+
@driver = driver
|
10
|
+
@options = options
|
11
|
+
end
|
12
|
+
|
13
|
+
def export
|
14
|
+
crontabs_by_host, libexec_by_host = @driver.export_crontab
|
15
|
+
parse(crontabs_by_host, libexec_by_host)
|
16
|
+
end
|
17
|
+
|
18
|
+
private
|
19
|
+
|
20
|
+
def parse(crontabs_by_host, libexec_by_host)
|
21
|
+
crontabs_by_host.each do |host, crontab_by_user|
|
22
|
+
libexec_contents = libexec_by_host[host] || {}
|
23
|
+
|
24
|
+
crontab_by_user.keys.each do |user|
|
25
|
+
crontab = crontab_by_user[user]
|
26
|
+
crontab_by_user[user] = parse_crontab(crontab, libexec_contents)
|
27
|
+
end
|
28
|
+
end
|
29
|
+
|
30
|
+
crontabs_by_host
|
31
|
+
end
|
32
|
+
|
33
|
+
def parse_crontab(crontab, libexec_contents)
|
34
|
+
scripts = {}
|
35
|
+
libexec_dir = @options.fetch(:libexec)
|
36
|
+
|
37
|
+
crontab.each_line.map(&:strip).each do |line|
|
38
|
+
next if line =~ /\A#/
|
39
|
+
|
40
|
+
md = line.match(/\A(@\w+|\S+(?:\s+\S+){4})\s+(.\S+)(.*)\z/)
|
41
|
+
schedule, path, extra = md.captures if md
|
42
|
+
|
43
|
+
if %r|\A#{Regexp.escape(libexec_dir)}/(?:[^/]+)/(.+)| =~ path
|
44
|
+
name = $1
|
45
|
+
|
46
|
+
scripts[name] = {
|
47
|
+
:schedule => schedule,
|
48
|
+
:path => path,
|
49
|
+
:content => libexec_contents[path]
|
50
|
+
}
|
51
|
+
end
|
52
|
+
end
|
53
|
+
|
54
|
+
scripts
|
55
|
+
end
|
56
|
+
end
|