cronicle 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,161 @@
1
+ class Cronicle::Client
2
+ include Cronicle::Logger::Helper
3
+
4
+ DEFAULTS = {
5
+ :concurrency => 10,
6
+ :libexec => '/var/lib/cronicle/libexec'
7
+ }
8
+
9
+ def initialize(host_list, options = {})
10
+ @host_list = host_list
11
+ @options = DEFAULTS.merge(options)
12
+ end
13
+
14
+ def apply(file)
15
+ walk(file)
16
+ end
17
+
18
+ def exec(file, name)
19
+ name = name.to_s
20
+ jobs = load_file(file)
21
+ jobs_by_host = select_host(jobs, name)
22
+
23
+ if jobs_by_host.empty?
24
+ raise "Definition cannot be found: Job `#{name}`"
25
+ end
26
+
27
+ parallel_each(jobs_by_host) do |host, jobs_by_user|
28
+ run_driver(host) do |driver|
29
+ jobs_by_user.each do |user, jobs|
30
+ driver.execute_job(user, jobs)
31
+ end
32
+ end
33
+ end
34
+ end
35
+
36
+ private
37
+
38
+ def walk(file)
39
+ jobs = load_file(file)
40
+ jobs_by_host = select_host(jobs)
41
+ exported = export_cron(jobs_by_host.keys)
42
+ walk_hosts(jobs_by_host, exported)
43
+ end
44
+
45
+ def export_cron(host_list)
46
+ driver = Cronicle::Driver.new(host_list, @options)
47
+ Cronicle::Exporter.export(driver, @options)
48
+ end
49
+
50
+ def walk_hosts(jobs_by_host, exported)
51
+ parallel_each(jobs_by_host) do |host, jobs_by_user|
52
+ exported_by_user = exported.delete(host) || {}
53
+ walk_host(host, jobs_by_user, exported_by_user)
54
+ end
55
+
56
+ parallel_each(exported) do |host, exported_by_user|
57
+ run_driver(host) do |driver|
58
+ scripts_by_user = {}
59
+
60
+ exported_by_user.each do |user, scripts|
61
+ scripts_by_user[user] = scripts
62
+ end
63
+
64
+ driver.delete_job(scripts_by_user)
65
+ end
66
+ end
67
+ end
68
+
69
+ def walk_host(host, jobs_by_user, exported_by_user)
70
+ run_driver(host) do |driver|
71
+ jobs_by_user.each do |user, jobs|
72
+ scripts = exported_by_user.delete(user) || {}
73
+ walk_jobs(driver, user, jobs, scripts)
74
+ end
75
+
76
+ exported_by_user.each do |user, scripts|
77
+ driver.delete_job(user => scripts)
78
+ end
79
+ end
80
+ end
81
+
82
+ def walk_jobs(driver, user, jobs, cron_cmds)
83
+ jobs.each do |name, job|
84
+ next unless job[:schedule]
85
+ current_cmd = cron_cmds.delete(name)
86
+
87
+ if current_cmd
88
+ driver.update_job(user, name, job, current_cmd)
89
+ else
90
+ driver.create_job(user, name, job)
91
+ end
92
+ end
93
+
94
+ cron_cmds.each do |name, current_cmd|
95
+ driver.delete_job(user => {name => current_cmd})
96
+ end
97
+ end
98
+
99
+ def run_driver(host)
100
+ driver = Cronicle::Driver.new(Array(host), @options)
101
+
102
+ if driver.test_sudo
103
+ yield(driver)
104
+ end
105
+ end
106
+
107
+ def select_host(jobs, target_name = nil)
108
+ hosts = Hash.new do |jobs_by_host, host|
109
+ jobs_by_host[host] = Hash.new do |jobs_by_user, user|
110
+ jobs_by_user[user] = {}
111
+ end
112
+ end
113
+
114
+ jobs.each do |job|
115
+ job_hash = job[:job]
116
+ job_user = job_hash[:user]
117
+ job_name = job_hash[:name]
118
+ servers = job[:servers]
119
+
120
+ if target_name and job_name != target_name
121
+ next
122
+ end
123
+
124
+ selected_hots = @host_list.select(
125
+ :servers => servers,
126
+ :roles => job[:roles]
127
+ )
128
+
129
+ # Add hosts that is defined in DSL
130
+ dsl_hosts = servers.select {|srvr|
131
+ srvr.kind_of?(String) or srvr.kind_of?(Symbol)
132
+ }.map(&:to_s)
133
+
134
+ (selected_hots + dsl_hosts).uniq.each do |h|
135
+ if hosts[h][job_user][job_name]
136
+ log(:warn, "Job is duplicated", :color => :yellow, :host => h, :user => job_user, :job => job_name)
137
+ else
138
+ hosts[h][job_user][job_name] = job_hash
139
+ end
140
+ end
141
+ end
142
+
143
+ hosts
144
+ end
145
+
146
+ def load_file(file)
147
+ if file.kind_of?(String)
148
+ open(file) do |f|
149
+ Cronicle::DSL.parse(f.read, file, @options)
150
+ end
151
+ elsif [File, Tempfile].any? {|i| file.kind_of?(i) }
152
+ Cronicle::DSL.parse(file.read, file.path, @options)
153
+ else
154
+ raise TypeError, "Cannot convert #{file} into File"
155
+ end
156
+ end
157
+
158
+ def parallel_each(enum, &block)
159
+ Parallel.each(enum, :in_threads => @options[:concurrency], &block)
160
+ end
161
+ end
@@ -0,0 +1,157 @@
1
+ class Cronicle::Driver
2
+ CRON_DIRS = %w(/var/spool/cron/crontabs /var/spool/cron)
3
+
4
+ attr_reader :hosts
5
+
6
+ def initialize(hosts, options = nil)
7
+ @hosts = hosts
8
+ @options = options
9
+ end
10
+
11
+ def execute(&block)
12
+ coordinator = SSHKit::Coordinator.new(@hosts)
13
+ hosts = coordinator.hosts
14
+
15
+ hosts.each do |host|
16
+ host.instance_variable_set(:@options, @options)
17
+ end
18
+
19
+ runner_opts = @options[:runner_options] || {}
20
+ runner = SSHKit::Runner::Group.new(hosts, runner_opts, &block)
21
+ runner.group_size = @options[:concurrency]
22
+ runner.execute
23
+ end
24
+
25
+ def export_crontab
26
+ crontabs_by_host = {}
27
+ libexec_by_host = {}
28
+
29
+ execute do
30
+ crontabs_by_host[host.hostname] = fetch_crontabs
31
+ libexec_by_host[host.hostname] = fetch_libexec_scripts
32
+ end
33
+
34
+ [crontabs_by_host, libexec_by_host]
35
+ end
36
+
37
+ def execute_job(user, jobs)
38
+ execute do
39
+ mktemp do |temp_dir|
40
+ jobs.each do |name, job|
41
+ log_opts = {:color => :cyan, :host => host.hostname, :user => user, :job => name}
42
+ log_msg = 'Execute job'
43
+
44
+ if host.options[:dry_run]
45
+ if host.options[:verbose]
46
+ content = job[:content].each_line.map {|l| ' ' + l }.join
47
+ log_msg << "\n" << content.chomp << "\n"
48
+ end
49
+
50
+ log_for_cronicle(:info, log_msg, log_opts)
51
+ next
52
+ else
53
+ log_for_cronicle(:info, log_msg, log_opts)
54
+ end
55
+
56
+ upload_script(temp_dir, name, job[:content]) do |temp_script|
57
+ command = sudo(:_execute, temp_script, :user => user, :raise_on_non_zero_exit => false)
58
+ out = command.full_stdout
59
+ Cronicle::Utils.remove_prompt!(out)
60
+ host_user_job = {:host => host.hostname, :user => user, :job => name}
61
+
62
+ put_log = proc do |level, opts|
63
+ opts ||= {}
64
+
65
+ out.each_line do |line|
66
+ log_for_cronicle(:info, line.strip, opts.merge(host_user_job))
67
+ end
68
+ end
69
+
70
+ if command.exit_status.zero?
71
+ put_log.call(:info)
72
+ else
73
+ put_log.call(:error, :color => :red)
74
+ end
75
+ end
76
+ end
77
+ end
78
+ end
79
+ end
80
+
81
+ def create_job(user, name, job)
82
+ create_or_update_job(user, name, job)
83
+ end
84
+
85
+ def update_job(user, name, job, script)
86
+ job_content = job[:content].chomp
87
+ script_content = script[:content].chomp
88
+
89
+ if [:schedule, :content].any? {|k| job[k].chomp != script[k].chomp }
90
+ create_or_update_job(user, name, job, script)
91
+ end
92
+ end
93
+
94
+ def create_or_update_job(user, name, job, script = nil)
95
+ execute do
96
+ log_opts = {:host => host.hostname, :user => user, :job => name}
97
+ log_opts[:color] = script ? :green : :cyan
98
+ log_msg = (script ? 'Update' : 'Create') + " job: schedule=#{job[:schedule].inspect}"
99
+
100
+ if host.options[:verbose]
101
+ content_orig = script ? script[:content] : ''
102
+ log_msg << "\n" << Cronicle::Utils.diff(content_orig, job[:content])
103
+ end
104
+
105
+ log_for_cronicle(:info, log_msg, log_opts)
106
+
107
+ unless host.options[:dry_run]
108
+ mktemp(user) do |temp_dir, user_temp_dir|
109
+ libexec_script = script_path(user, name)
110
+
111
+ upload_script(temp_dir, name, job[:content]) do |temp_script|
112
+ temp_entry = temp_script + '.entry'
113
+ sudo(:execute, :mkdir, '-p', user_libexec_dir(user))
114
+ sudo(:execute, :cp, temp_script, libexec_script)
115
+ sudo(:execute, :touch, user_crontab(user))
116
+ delete_cron_entry(user, name)
117
+ add_cron_entry(user, name, job[:schedule], user_temp_dir)
118
+ end
119
+ end
120
+ end
121
+ end
122
+ end
123
+
124
+ def delete_job(scripts_by_user, target_name = nil)
125
+ execute do
126
+ scripts_by_user.each do |user, scripts|
127
+ scripts.each do |name, script|
128
+ next if target_name && target_name != name
129
+
130
+ log_opts = {:color => :red, :host => host.hostname, :user => user, :job => name}
131
+ log_msg = "Delete job: schedule=#{script[:schedule].inspect}"
132
+ log_msg << "\n" << Cronicle::Utils.diff(script[:content], '') if host.options[:verbose]
133
+ log_for_cronicle(:info, log_msg, log_opts)
134
+
135
+ unless host.options[:dry_run]
136
+ delete_cron_entry(user, name)
137
+ sudo(:execute, :rm, '-f', script[:path], :raise_on_non_zero_exit => false)
138
+ end
139
+ end
140
+ end
141
+ end
142
+ end
143
+
144
+ def test_sudo
145
+ ok = false
146
+
147
+ execute do
148
+ ok = sudo(:execute, :echo, :raise_on_non_zero_exit => false)
149
+
150
+ unless ok
151
+ log_for_cronicle(:error, 'incorrect sudo password', :color => :red, :host => host.hostname)
152
+ end
153
+ end
154
+
155
+ ok
156
+ end
157
+ end
@@ -0,0 +1,7 @@
1
+ class Cronicle::DSL
2
+ class << self
3
+ def parse(dsl, path, opts = {})
4
+ Cronicle::DSL::Context.eval(dsl, path, opts).result
5
+ end
6
+ end # of class methods
7
+ end
@@ -0,0 +1,48 @@
1
+ class Cronicle::DSL::Context
2
+ class << self
3
+ def eval(dsl, path, opts = {})
4
+ self.new(path, opts) {
5
+ Kernel.eval(dsl, binding, path)
6
+ }
7
+ end
8
+ end # of class methods
9
+
10
+ attr_reader :result
11
+
12
+ def initialize(path, options = {}, &block)
13
+ @path = path
14
+ @options = options
15
+ @result = []
16
+ instance_eval(&block)
17
+ end
18
+
19
+ def require(file)
20
+ cronfile = (file =~ %r|\A/|) ? file : File.expand_path(File.join(File.dirname(@path), file))
21
+
22
+ if File.exist?(cronfile)
23
+ instance_eval(File.read(cronfile), cronfile)
24
+ elsif File.exist?(cronfile + '.rb')
25
+ instance_eval(File.read(cronfile + '.rb'), cronfile + '.rb')
26
+ else
27
+ Kernel.require(file)
28
+ end
29
+ end
30
+
31
+ def on(target, &block)
32
+ unless block
33
+ raise ArgumentError, "Block is required for `on` method"
34
+ end
35
+
36
+ unless target.kind_of?(Hash)
37
+ raise TypeError, "wrong argument type #{target.class} (expected Hash)"
38
+ end
39
+
40
+ if target.empty?
41
+ raise ArgumentError, ':servers or :roles is not passed to `on` method'
42
+ end
43
+
44
+ target.assert_valid_keys(:servers, :roles)
45
+
46
+ @result.concat(Cronicle::DSL::Context::Job.new(target, &block).result.values)
47
+ end
48
+ end
@@ -0,0 +1,58 @@
1
+ class Cronicle::DSL::Context::Job
2
+ def initialize(target, &block)
3
+ @result = Hash.new {|hash, key|
4
+ hash[key] = {
5
+ :servers => Array(target[:servers]),
6
+ :roles => Array(target[:roles]),
7
+ :job => {}
8
+ }
9
+ }
10
+
11
+ instance_eval(&block)
12
+ end
13
+
14
+ attr_reader :result
15
+
16
+ def job(name, opts = {}, &block)
17
+ name = name.to_s
18
+
19
+ raise ArgumentError, %!Job name is required! if (name || '').strip.empty?
20
+
21
+ if @result.has_key?(name)
22
+ raise "Job `#{name}`: already defined"
23
+ end
24
+
25
+ unless opts.kind_of?(Hash)
26
+ raise TypeError, "Job `#{name}`: wrong argument type #{opts.class} (expected Hash)"
27
+ end
28
+
29
+ unless opts[:user]
30
+ raise ArgumentError, "Job `#{name}`: :user is required"
31
+ end
32
+
33
+ opts.assert_valid_keys(:schedule, :user, :content)
34
+
35
+ if opts[:content] and block
36
+ raise ArgumentError, 'Can not pass :content and block to `job` method'
37
+ elsif not opts[:content] and not block
38
+ raise ArgumentError, "Job `#{name}`: :context or block is required"
39
+ end
40
+
41
+ job_hash = @result[name][:job]
42
+ job_hash[:name] = name
43
+ job_hash[:user] = opts.fetch(:user).to_s
44
+ job_hash[:schedule] = opts[:schedule].to_s if opts[:schedule]
45
+
46
+ if block
47
+ source = block.to_raw_source(:strip_enclosure => true).each_line.to_a
48
+ source = source.shift + source.join.undent
49
+
50
+ job_hash[:content] = <<-RUBY
51
+ #!/usr/bin/env ruby
52
+ #{source}
53
+ RUBY
54
+ else
55
+ job_hash[:content] = opts[:content].to_s.undent
56
+ end
57
+ end
58
+ end
@@ -0,0 +1,56 @@
1
+ class Cronicle::Exporter
2
+ class << self
3
+ def export(driver, opts = {})
4
+ self.new(driver, opts).export
5
+ end
6
+ end # of class methods
7
+
8
+ def initialize(driver, options = {})
9
+ @driver = driver
10
+ @options = options
11
+ end
12
+
13
+ def export
14
+ crontabs_by_host, libexec_by_host = @driver.export_crontab
15
+ parse(crontabs_by_host, libexec_by_host)
16
+ end
17
+
18
+ private
19
+
20
+ def parse(crontabs_by_host, libexec_by_host)
21
+ crontabs_by_host.each do |host, crontab_by_user|
22
+ libexec_contents = libexec_by_host[host] || {}
23
+
24
+ crontab_by_user.keys.each do |user|
25
+ crontab = crontab_by_user[user]
26
+ crontab_by_user[user] = parse_crontab(crontab, libexec_contents)
27
+ end
28
+ end
29
+
30
+ crontabs_by_host
31
+ end
32
+
33
+ def parse_crontab(crontab, libexec_contents)
34
+ scripts = {}
35
+ libexec_dir = @options.fetch(:libexec)
36
+
37
+ crontab.each_line.map(&:strip).each do |line|
38
+ next if line =~ /\A#/
39
+
40
+ md = line.match(/\A(@\w+|\S+(?:\s+\S+){4})\s+(.\S+)(.*)\z/)
41
+ schedule, path, extra = md.captures if md
42
+
43
+ if %r|\A#{Regexp.escape(libexec_dir)}/(?:[^/]+)/(.+)| =~ path
44
+ name = $1
45
+
46
+ scripts[name] = {
47
+ :schedule => schedule,
48
+ :path => path,
49
+ :content => libexec_contents[path]
50
+ }
51
+ end
52
+ end
53
+
54
+ scripts
55
+ end
56
+ end