dply 0.1.13 → 0.1.17
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/bin/dplyr +9 -1
- data/bin/drake +3 -1
- data/docs/stages.rb +9 -4
- data/lib/dply/cli/devbuild.rb +67 -0
- data/lib/dply/cli/setup.rb +102 -0
- data/lib/dply/config.rb +4 -0
- data/lib/dply/config_struct.rb +1 -1
- data/lib/dply/custom_logger.rb +6 -1
- data/lib/dply/git.rb +6 -1
- data/lib/dply/pkgs_config.rb +1 -1
- data/lib/dply/repo.rb +1 -1
- data/lib/dply/setup.rb +1 -1
- data/lib/dply/tasks.rb +29 -5
- data/lib/dply/templates/build.erb +5 -0
- data/lib/dply/templates/deploy.erb +15 -0
- data/lib/dply/templates/pkgs.erb +5 -0
- data/lib/dply/version.rb +1 -1
- data/lib/dply/yum.rb +4 -4
- data/lib/dplyr/cli.rb +3 -3
- data/lib/dplyr/remote_task.rb +45 -111
- data/lib/dplyr/report.rb +4 -1
- data/lib/dplyr/stage.rb +24 -3
- data/lib/dplyr/task_runner.rb +129 -0
- metadata +9 -3
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: e36e026a2a88ee74042f732298e42fb7f13a4eca
|
4
|
+
data.tar.gz: b82102df075f09482830a6e48850710a6788b09a
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: e0d0e00e407aab69f5394aad3dc2e8833c41db1c2869eff653f5dd396cb681f1fd017e8211bf7795c9f1ce9001fba5b5f8885b113844bf7eae9d66b89ec999e8
|
7
|
+
data.tar.gz: 27983c97231765d2b2387c02e1f234fd8b060fcd2652389047be8cbeeb05cbd1e19e98380b8de502d2db0161740067a2dc3ef691f6733c2972dcf0367f495991
|
data/bin/dplyr
CHANGED
@@ -45,10 +45,14 @@ opts_parser = OptionParser.new do |opts|
|
|
45
45
|
logger.level = ::Logger::DEBUG
|
46
46
|
end
|
47
47
|
|
48
|
+
opts.on("-m", "--enable-markers", "Enable marker logging") do
|
49
|
+
logger.enable_markers = true
|
50
|
+
end
|
51
|
+
|
48
52
|
opts.on_tail("-h", "--help", "Show this message") do
|
49
53
|
puts opts
|
50
54
|
exit
|
51
|
-
end
|
55
|
+
end
|
52
56
|
end
|
53
57
|
begin
|
54
58
|
opts_parser.order!(ARGV)
|
@@ -70,4 +74,8 @@ rescue OptionParser::InvalidOption, OptionParser::MissingArgument, ::Dply::Error
|
|
70
74
|
logger.error "#{e.message}"
|
71
75
|
abort
|
72
76
|
end
|
77
|
+
ensure
|
78
|
+
logger.marker "end"
|
73
79
|
end
|
80
|
+
|
81
|
+
|
data/bin/drake
CHANGED
@@ -71,7 +71,9 @@ begin
|
|
71
71
|
|
72
72
|
command = (ARGV.shift || "").to_sym
|
73
73
|
case command
|
74
|
-
when :deploy, :reload, :task,
|
74
|
+
when :deploy, :reload, :task,
|
75
|
+
:build, :depcheck, :"install-pkgs", :reopen_logs,
|
76
|
+
:status, :devbuild, :setup
|
75
77
|
run_cli command, ARGV
|
76
78
|
when /\A[a-zA-Z_\-0-9]+[:][a-zA-Z_\-0-9]+\z/
|
77
79
|
require 'dply/tasks'
|
data/docs/stages.rb
CHANGED
@@ -1,8 +1,13 @@
|
|
1
|
+
# Sample stages.rb
|
2
|
+
# __________________________________________________________
|
3
|
+
#
|
4
|
+
|
1
5
|
stage :production do
|
2
6
|
user "dev"
|
3
|
-
|
4
|
-
deploy_dir "/home/dev/project"
|
7
|
+
dir "/home/dev/project"
|
5
8
|
parallel_runs 1
|
6
|
-
host "10.1.1.1",
|
7
|
-
host "10.1.1.2",
|
9
|
+
host "10.1.1.1", dir: "/home/dev/project1", id: "project1", user: "dev1" ## default roles: ["first"]
|
10
|
+
host "10.1.1.2", dir: "/home/dev/project2", id: "project2"
|
11
|
+
host "10.1.1.3", roles: ["db"]
|
12
|
+
host "10.1.1.4" ## default roles: ["last"]
|
8
13
|
end
|
@@ -0,0 +1,67 @@
|
|
1
|
+
require 'dply/logger'
|
2
|
+
require 'dply/lock'
|
3
|
+
require 'dply/tasks'
|
4
|
+
require 'fileutils'
|
5
|
+
|
6
|
+
module Dply
|
7
|
+
module Cli
|
8
|
+
class Devbuild
|
9
|
+
|
10
|
+
include Logger
|
11
|
+
|
12
|
+
def initialize(argv)
|
13
|
+
@argv = argv
|
14
|
+
@options = {}
|
15
|
+
end
|
16
|
+
|
17
|
+
def run
|
18
|
+
lock.acquire
|
19
|
+
opts.parse!(@argv)
|
20
|
+
revision = @options[:revision] || "dev"
|
21
|
+
ENV["BUILD_NUMBER"] = revision
|
22
|
+
|
23
|
+
build_artifacts = "tmp/build_artifacts"
|
24
|
+
FileUtils.mkdir_p build_artifacts
|
25
|
+
|
26
|
+
clear_bundle_config
|
27
|
+
tasks.install_pkgs(build_mode: true, use_yum: @options[:use_yum])
|
28
|
+
tasks.build "build:default"
|
29
|
+
ensure
|
30
|
+
clear_bundle_config
|
31
|
+
end
|
32
|
+
|
33
|
+
def tasks
|
34
|
+
@tasks ||= ::Dply::Tasks.new
|
35
|
+
end
|
36
|
+
|
37
|
+
def lock
|
38
|
+
@lock ||= ::Dply::Lock.new
|
39
|
+
end
|
40
|
+
|
41
|
+
def clear_bundle_config
|
42
|
+
FileUtils.rm ".bundle/config" if File.exists? ".bundle/config"
|
43
|
+
end
|
44
|
+
|
45
|
+
def opts
|
46
|
+
OptionParser.new do |opts|
|
47
|
+
|
48
|
+
opts.banner = "Usage: drake devbuild [options] [target]"
|
49
|
+
|
50
|
+
opts.on("-r", "--revision [REVISION]", "Specify revision") do |r|
|
51
|
+
@options[:revision] = r
|
52
|
+
end
|
53
|
+
|
54
|
+
opts.on("--use-yum", "use yum to install packages") do |e|
|
55
|
+
@options[:use_yum] = true
|
56
|
+
end
|
57
|
+
|
58
|
+
opts.on("-h", "--help", "Help") do
|
59
|
+
puts opts
|
60
|
+
exit
|
61
|
+
end
|
62
|
+
end
|
63
|
+
end
|
64
|
+
|
65
|
+
end
|
66
|
+
end
|
67
|
+
end
|
@@ -0,0 +1,102 @@
|
|
1
|
+
require 'dply/logger'
|
2
|
+
require 'dply/lock'
|
3
|
+
require 'fileutils'
|
4
|
+
require 'erb'
|
5
|
+
|
6
|
+
module Dply
|
7
|
+
module Cli
|
8
|
+
class Setup
|
9
|
+
|
10
|
+
include ::Dply::Logger
|
11
|
+
|
12
|
+
def initialize(argv)
|
13
|
+
@argv = argv
|
14
|
+
@options = {}
|
15
|
+
end
|
16
|
+
|
17
|
+
def run
|
18
|
+
opts.parse!(@argv)
|
19
|
+
return if not proceed?
|
20
|
+
lock.acquire
|
21
|
+
setup_default
|
22
|
+
namespace = @argv.shift
|
23
|
+
if namespace
|
24
|
+
setup_task(namespace)
|
25
|
+
else
|
26
|
+
setup_build_task
|
27
|
+
setup_task(:production)
|
28
|
+
end
|
29
|
+
end
|
30
|
+
|
31
|
+
def proceed?
|
32
|
+
print "Are you sure?(y/n) "
|
33
|
+
v = STDIN.gets.strip
|
34
|
+
v == "y"
|
35
|
+
end
|
36
|
+
|
37
|
+
def setup_default
|
38
|
+
if not File.exist? "dply"
|
39
|
+
FileUtils.mkdir_p "dply"
|
40
|
+
logger.info "created dply/ dir"
|
41
|
+
else
|
42
|
+
logger.info "skipping dply/ dir"
|
43
|
+
end
|
44
|
+
|
45
|
+
rakefile = "dply/Rakefile"
|
46
|
+
if not File.exist? rakefile
|
47
|
+
FileUtils.touch rakefile
|
48
|
+
logger.info "created #{rakefile}"
|
49
|
+
else
|
50
|
+
logger.info "skipping #{rakefile}"
|
51
|
+
end
|
52
|
+
|
53
|
+
pkgs_yml = "pkgs.yml"
|
54
|
+
if not File.exist? pkgs_yml
|
55
|
+
FileUtils.cp "#{templates_dir}/pkgs.erb", pkgs_yml
|
56
|
+
logger.info "created #{pkgs_yml}"
|
57
|
+
else
|
58
|
+
logger.info "skipping #{pkgs_yml}"
|
59
|
+
end
|
60
|
+
end
|
61
|
+
|
62
|
+
def setup_task(namespace, template: "deploy.erb")
|
63
|
+
tasks_file = "dply/#{namespace}.rake"
|
64
|
+
if File.exist? tasks_file
|
65
|
+
logger.info "skipping #{tasks_file}"
|
66
|
+
return
|
67
|
+
end
|
68
|
+
b = binding
|
69
|
+
template_path = "#{templates_dir}/#{template}"
|
70
|
+
erb = ERB.new(File.read(template_path), nil, '-')
|
71
|
+
File.open(tasks_file, 'w') { |f| f.write erb.result(b) }
|
72
|
+
logger.info "created #{tasks_file}"
|
73
|
+
end
|
74
|
+
|
75
|
+
def setup_build_task
|
76
|
+
setup_task("build", template: "build.erb")
|
77
|
+
end
|
78
|
+
|
79
|
+
def templates_dir
|
80
|
+
@templates_dir ||= "#{__dir__}/../templates"
|
81
|
+
end
|
82
|
+
|
83
|
+
def lock
|
84
|
+
@lock ||= ::Dply::Lock.new
|
85
|
+
end
|
86
|
+
|
87
|
+
def opts
|
88
|
+
OptionParser.new do |opts|
|
89
|
+
|
90
|
+
opts.banner = "Usage: drake setup [options] [namespace]"
|
91
|
+
|
92
|
+
opts.on("-h", "--help", "Help") do
|
93
|
+
puts opts
|
94
|
+
exit
|
95
|
+
end
|
96
|
+
end
|
97
|
+
end
|
98
|
+
|
99
|
+
|
100
|
+
end
|
101
|
+
end
|
102
|
+
end
|
data/lib/dply/config.rb
CHANGED
data/lib/dply/config_struct.rb
CHANGED
@@ -4,7 +4,7 @@ module Dply
|
|
4
4
|
|
5
5
|
attr_writer :revision, :build_url, :build_url_proc,
|
6
6
|
:revision_proc
|
7
|
-
attr_accessor :dir, :name, :repo, :branch,
|
7
|
+
attr_accessor :dir, :name, :repo, :branch, :mirror,
|
8
8
|
:strategy, :target, :verify_checksum,
|
9
9
|
:config_map, :dir_map, :shared_dirs,
|
10
10
|
:config_download_url, :config_skip_download
|
data/lib/dply/custom_logger.rb
CHANGED
@@ -3,7 +3,7 @@ require 'dply/ext/string'
|
|
3
3
|
module Dply
|
4
4
|
class CustomLogger < ::Logger
|
5
5
|
|
6
|
-
attr_writer :trace_mode, :remote_mode
|
6
|
+
attr_writer :trace_mode, :remote_mode, :enable_markers
|
7
7
|
|
8
8
|
def initialize(file)
|
9
9
|
super(file)
|
@@ -37,5 +37,10 @@ module Dply
|
|
37
37
|
puts %{dply_msg|#{msg}}
|
38
38
|
end
|
39
39
|
|
40
|
+
def marker(msg)
|
41
|
+
return if not @enable_markers
|
42
|
+
puts "dply_marker:#{msg}"
|
43
|
+
end
|
44
|
+
|
40
45
|
end
|
41
46
|
end
|
data/lib/dply/git.rb
CHANGED
@@ -20,7 +20,12 @@ module Dply
|
|
20
20
|
end
|
21
21
|
|
22
22
|
def self.clone(repo, dir, mirror: nil)
|
23
|
-
|
23
|
+
if mirror
|
24
|
+
cmd "git clone #{mirror} #{dir}"
|
25
|
+
Dir.chdir(dir) { cmd "git remote set-url origin #{repo}" }
|
26
|
+
else
|
27
|
+
cmd "git clone #{repo} #{dir}"
|
28
|
+
end
|
24
29
|
end
|
25
30
|
|
26
31
|
def self.clean
|
data/lib/dply/pkgs_config.rb
CHANGED
data/lib/dply/repo.rb
CHANGED
data/lib/dply/setup.rb
CHANGED
data/lib/dply/tasks.rb
CHANGED
@@ -4,6 +4,8 @@ require 'dply/bundle'
|
|
4
4
|
require 'dply/yum'
|
5
5
|
require 'dply/linker'
|
6
6
|
require 'dply/pkgs_config'
|
7
|
+
require 'dply/error'
|
8
|
+
require 'etc'
|
7
9
|
|
8
10
|
module Dply
|
9
11
|
class Tasks
|
@@ -49,9 +51,12 @@ module Dply
|
|
49
51
|
def install_pkgs(build_mode: false, use_yum: false)
|
50
52
|
return if not File.exists? "pkgs.yml"
|
51
53
|
drake_exists = File.exists? (drake_command)
|
54
|
+
|
55
|
+
pkgs = get_pkgs(build_mode)
|
52
56
|
if use_yum || !drake_exists
|
53
|
-
yum_install
|
57
|
+
yum_install pkgs
|
54
58
|
else
|
59
|
+
return if Yum.new(pkgs).installed?
|
55
60
|
command_install build_mode
|
56
61
|
end
|
57
62
|
end
|
@@ -70,20 +75,39 @@ module Dply
|
|
70
75
|
@bundle ||= Bundle.new(deployment: @deployment)
|
71
76
|
end
|
72
77
|
|
73
|
-
def
|
74
|
-
|
78
|
+
def get_pkgs(build_mode)
|
79
|
+
PkgsConfig.new(build_mode: build_mode).pkgs
|
80
|
+
end
|
81
|
+
|
82
|
+
def yum_install(pkgs)
|
75
83
|
Yum.new(pkgs, sudo: true).install
|
76
84
|
end
|
77
85
|
|
78
86
|
def command_install(build_mode)
|
79
|
-
command = "
|
87
|
+
command = "#{drake_command} install-pkgs"
|
80
88
|
command << " -b" if build_mode
|
81
|
-
|
89
|
+
check_sudo_permission command
|
90
|
+
cmd "sudo -n #{command}"
|
82
91
|
end
|
83
92
|
|
84
93
|
def drake_command
|
85
94
|
@drake_command ||= (ENV["DRAKE_COMMAND"] || "/opt/ruby/bin/drake")
|
86
95
|
end
|
87
96
|
|
97
|
+
def check_sudo_permission(command)
|
98
|
+
output = `sudo -n -l #{command}`
|
99
|
+
if output.chomp.strip == command
|
100
|
+
return true
|
101
|
+
else
|
102
|
+
msg = []
|
103
|
+
user = Etc.getpwuid(Process.uid).name
|
104
|
+
msg << %{unable to run "#{command}" with sudo permissions}
|
105
|
+
msg << %{To resolve add the following line to sudoers: }
|
106
|
+
msg << %{#{user} ALL=(ALL) NOPASSWD: /opt/ruby/bin/drake install-pkgs *, /opt/ruby/bin/drake install-pkgs}.yellow
|
107
|
+
raise Error, msg.join("\n")
|
108
|
+
end
|
109
|
+
|
110
|
+
end
|
111
|
+
|
88
112
|
end
|
89
113
|
end
|
data/lib/dply/version.rb
CHANGED
data/lib/dply/yum.rb
CHANGED
@@ -26,6 +26,10 @@ module Dply
|
|
26
26
|
end
|
27
27
|
end
|
28
28
|
|
29
|
+
def installed?
|
30
|
+
not_installed_pkgs.size == 0
|
31
|
+
end
|
32
|
+
|
29
33
|
private
|
30
34
|
|
31
35
|
def pkgs_str
|
@@ -42,9 +46,5 @@ module Dply
|
|
42
46
|
matches = `#{command}`.scan(/^package (.*) is not installed$/)
|
43
47
|
end
|
44
48
|
|
45
|
-
def installed?
|
46
|
-
not_installed_pkgs.size == 0
|
47
|
-
end
|
48
|
-
|
49
49
|
end
|
50
50
|
end
|
data/lib/dplyr/cli.rb
CHANGED
@@ -1,5 +1,5 @@
|
|
1
1
|
require 'dplyr/stages_config'
|
2
|
-
require 'dplyr/
|
2
|
+
require 'dplyr/task_runner'
|
3
3
|
require 'dply/logger'
|
4
4
|
|
5
5
|
module Dplyr
|
@@ -52,8 +52,8 @@ module Dplyr
|
|
52
52
|
end
|
53
53
|
|
54
54
|
def run_remote_task
|
55
|
-
|
56
|
-
|
55
|
+
task_runner = TaskRunner.new(hosts, argv_str, parallel_jobs: parallel_jobs)
|
56
|
+
task_runner.run
|
57
57
|
end
|
58
58
|
|
59
59
|
end
|
data/lib/dplyr/remote_task.rb
CHANGED
@@ -6,85 +6,49 @@ module Dplyr
|
|
6
6
|
class RemoteTask
|
7
7
|
|
8
8
|
include ::Dply::Logger
|
9
|
-
|
10
|
-
attr_reader :hosts, :parallel_jobs, :task, :env
|
11
9
|
|
12
|
-
|
13
|
-
|
14
|
-
|
10
|
+
attr_reader :exit_status, :messages
|
11
|
+
|
12
|
+
def initialize(host, task, id_size: nil)
|
13
|
+
@host = host
|
15
14
|
@task = task
|
16
|
-
@
|
17
|
-
@
|
15
|
+
@messages = []
|
16
|
+
@id_size = id_size || @host_info[:id].size
|
18
17
|
end
|
19
18
|
|
20
19
|
def run
|
21
|
-
|
22
|
-
|
23
|
-
|
24
|
-
|
25
|
-
end
|
20
|
+
reset!
|
21
|
+
r, w, pid = PTY.spawn(remote_cmd)
|
22
|
+
pty_read(r, @host[:id])
|
23
|
+
@exit_status = get_exit_status pid
|
26
24
|
end
|
27
25
|
|
28
|
-
def
|
29
|
-
|
30
|
-
|
31
|
-
|
32
|
-
|
33
|
-
loop do
|
34
|
-
break if @queue.empty?
|
35
|
-
@mq.pop
|
36
|
-
spawn_queued_job
|
37
|
-
end
|
38
|
-
report.print_full
|
39
|
-
end
|
40
|
-
|
41
|
-
def run_serially
|
42
|
-
hosts.each do |host_info|
|
43
|
-
puts "=== Running on #{host_info[:id]} ==="
|
44
|
-
run_cmd = remote_cmd host_info
|
45
|
-
system run_cmd
|
46
|
-
puts
|
47
|
-
raise ::Dply::Error, "remote deploy failed on #{host_info[:id]}" if $? != 0
|
26
|
+
def remote_cmd
|
27
|
+
if logger.debug?
|
28
|
+
%(#{ssh} -l #{user} #{addr} '#{env} drake --remote --debug -d #{dir} #{@task} 2>&1')
|
29
|
+
else
|
30
|
+
%(#{ssh} -l #{user} #{addr} '#{env} drake --remote -d #{dir} #{@task} 2>&1' 2>/dev/null)
|
48
31
|
end
|
49
32
|
end
|
50
33
|
|
51
|
-
|
52
|
-
|
53
|
-
def init_run
|
54
|
-
queue_all_hosts
|
55
|
-
job_output_template
|
56
|
-
@threads = {}
|
57
|
-
@mq = Queue.new
|
34
|
+
def user
|
35
|
+
@host.fetch :user
|
58
36
|
end
|
59
37
|
|
60
|
-
def
|
61
|
-
@
|
62
|
-
hosts.each { |h| @queue << h }
|
38
|
+
def dir
|
39
|
+
@host.fetch :dir
|
63
40
|
end
|
64
41
|
|
65
|
-
def
|
66
|
-
|
67
|
-
thread = popen(run_cmd, host_info)
|
68
|
-
@threads[host_info] = thread
|
42
|
+
def addr
|
43
|
+
@host.fetch :addr
|
69
44
|
end
|
70
45
|
|
71
|
-
def
|
72
|
-
|
73
|
-
host = @queue.pop(false)
|
74
|
-
spawn_job host
|
75
|
-
rescue ThreadError
|
46
|
+
def roles
|
47
|
+
@host.fetch :roles
|
76
48
|
end
|
77
|
-
|
78
|
-
def
|
79
|
-
|
80
|
-
host = host_info[:host]
|
81
|
-
dir = host_info[:dir]
|
82
|
-
ssh = "ssh -tt -o BatchMode=yes -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null"
|
83
|
-
if logger.debug?
|
84
|
-
%(#{ssh} -l #{user} #{host} '#{env} drake --remote --debug -d #{dir} #{task} 2>&1')
|
85
|
-
else
|
86
|
-
%(#{ssh} -l #{user} #{host} '#{env} drake --remote -d #{dir} #{task} 2>&1' 2>/dev/null)
|
87
|
-
end
|
49
|
+
|
50
|
+
def ssh
|
51
|
+
"ssh -tt -o BatchMode=yes -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null"
|
88
52
|
end
|
89
53
|
|
90
54
|
def pty_read(file, id)
|
@@ -92,72 +56,42 @@ module Dplyr
|
|
92
56
|
if line =~ /\Adply_msg\|/
|
93
57
|
receive_message line
|
94
58
|
else
|
95
|
-
printf
|
59
|
+
printf output_template, id, line
|
96
60
|
end
|
97
61
|
end
|
98
62
|
rescue EOFError,Errno::ECONNRESET, Errno::EPIPE, Errno::EIO => e
|
99
63
|
end
|
100
64
|
|
101
|
-
def popen(cmd, host_info)
|
102
|
-
t = Thread.new do |t|
|
103
|
-
Thread.current[:messages] = []
|
104
|
-
begin
|
105
|
-
r, w, pid = PTY.spawn(cmd)
|
106
|
-
pty_read(r, host_info[:id])
|
107
|
-
exit_status pid
|
108
|
-
ensure
|
109
|
-
@mq << true
|
110
|
-
end
|
111
|
-
end
|
112
|
-
t.abort_on_exception = true
|
113
|
-
t.run
|
114
|
-
return t
|
115
|
-
end
|
116
|
-
|
117
|
-
def host_id_max_width
|
118
|
-
hosts.map {|h| h[:id].size }.max
|
119
|
-
end
|
120
|
-
|
121
|
-
def job_output_template
|
122
|
-
@job_output_template ||= begin
|
123
|
-
id_template = "%-#{host_id_max_width}s".bold.grey
|
124
|
-
template = "#{id_template} %s"
|
125
|
-
end
|
126
|
-
end
|
127
|
-
|
128
65
|
def receive_message(msg_str)
|
129
66
|
msg = msg_str.partition("|")[2].strip
|
130
67
|
return if msg.empty?
|
131
|
-
messages
|
132
|
-
messages << msg
|
68
|
+
@messages << msg
|
133
69
|
end
|
134
70
|
|
135
|
-
def
|
136
|
-
|
71
|
+
def get_exit_status(pid)
|
72
|
+
pid, status = Process.waitpid2(pid)
|
73
|
+
return status
|
137
74
|
end
|
138
75
|
|
139
|
-
def
|
140
|
-
@
|
141
|
-
|
142
|
-
|
143
|
-
m[host] = thread[:messages]
|
144
|
-
end
|
145
|
-
m
|
76
|
+
def output_template
|
77
|
+
@output_template ||= begin
|
78
|
+
id_template = "%-#{@id_size}s".bold.grey
|
79
|
+
template = "#{id_template} %s"
|
146
80
|
end
|
147
81
|
end
|
148
82
|
|
149
|
-
def
|
150
|
-
|
151
|
-
@exit_statuses = {}
|
152
|
-
@threads.each do |host, thread|
|
153
|
-
@exit_statuses[host] = thread.value
|
154
|
-
end
|
155
|
-
@exit_statuses
|
83
|
+
def reset!
|
84
|
+
@output_template = nil
|
156
85
|
end
|
157
86
|
|
158
|
-
def
|
159
|
-
|
160
|
-
|
87
|
+
def env
|
88
|
+
@env ||= "PATH=/usr/sbin:/usr/local/sbin:$PATH #{roles_env}"
|
89
|
+
end
|
90
|
+
|
91
|
+
def roles_env
|
92
|
+
return "" if not roles.size > 0
|
93
|
+
roles_str = roles.join(",")
|
94
|
+
"DPLY_ROLES=#{roles_str}"
|
161
95
|
end
|
162
96
|
|
163
97
|
end
|
data/lib/dplyr/report.rb
CHANGED
@@ -1,6 +1,8 @@
|
|
1
|
+
require 'dply/logger'
|
1
2
|
module Dplyr
|
2
3
|
class Report
|
3
4
|
|
5
|
+
include ::Dply::Logger
|
4
6
|
attr_reader :hosts, :exit_statuses
|
5
7
|
|
6
8
|
def initialize(hosts, exit_statuses, messages)
|
@@ -11,6 +13,7 @@ module Dplyr
|
|
11
13
|
|
12
14
|
|
13
15
|
def print_full
|
16
|
+
logger.marker "summary_start"
|
14
17
|
print_successful_jobs
|
15
18
|
print_failed_jobs
|
16
19
|
print_summary
|
@@ -52,7 +55,7 @@ module Dplyr
|
|
52
55
|
run_count = @exit_statuses.count
|
53
56
|
not_run = total_hosts - run_count
|
54
57
|
if (failed.count > 0 || not_run > 0 )
|
55
|
-
not_run_error = "not run on #{not_run}
|
58
|
+
not_run_error = "not run on #{not_run} of #{total_hosts} hosts" if not_run > 0
|
56
59
|
failed_error = "failed on #{failed.count} of #{total_hosts} hosts" if failed.count > 0
|
57
60
|
|
58
61
|
errors = []
|
data/lib/dplyr/stage.rb
CHANGED
@@ -1,6 +1,10 @@
|
|
1
|
+
require 'dply/helper'
|
2
|
+
|
1
3
|
module Dplyr
|
2
4
|
class Stage
|
3
5
|
|
6
|
+
include ::Dply::Helper
|
7
|
+
|
4
8
|
attr_accessor :config_proc
|
5
9
|
|
6
10
|
def initialize(name)
|
@@ -18,12 +22,13 @@ module Dplyr
|
|
18
22
|
})
|
19
23
|
end
|
20
24
|
|
21
|
-
def host(
|
25
|
+
def host(addr, user: nil, dir: nil, id: nil, roles: [])
|
22
26
|
@hosts << ({
|
23
|
-
|
27
|
+
addr: addr,
|
24
28
|
user: user,
|
25
29
|
dir: dir,
|
26
|
-
id: id ||
|
30
|
+
id: id || addr,
|
31
|
+
roles: cleaned_roles(roles)
|
27
32
|
})
|
28
33
|
end
|
29
34
|
|
@@ -54,15 +59,31 @@ module Dplyr
|
|
54
59
|
return value
|
55
60
|
end
|
56
61
|
|
62
|
+
def add_default_roles
|
63
|
+
return if not @hosts.size > 0
|
64
|
+
@hosts.first[:roles] << "first"
|
65
|
+
@hosts.last[:roles] << "last"
|
66
|
+
end
|
67
|
+
|
57
68
|
def finalize
|
58
69
|
return if @finalized
|
59
70
|
instance_eval &config_proc if config_proc
|
60
71
|
fill_hosts
|
72
|
+
add_default_roles
|
61
73
|
@hosts.freeze
|
62
74
|
@parallel_runs.freeze
|
63
75
|
@finalized = true
|
64
76
|
end
|
65
77
|
|
78
|
+
def cleaned_roles(roles)
|
79
|
+
error "roles must be an array" if not roles.is_a? Array
|
80
|
+
roles.each do |r|
|
81
|
+
r.strip!
|
82
|
+
error "invalid role value #{r} " if not r =~ /\A[0-9A-Za-z_\-]+\z/
|
83
|
+
end
|
84
|
+
return roles
|
85
|
+
end
|
86
|
+
|
66
87
|
end
|
67
88
|
end
|
68
89
|
|
@@ -0,0 +1,129 @@
|
|
1
|
+
require 'dplyr/report'
|
2
|
+
require 'dply/logger'
|
3
|
+
require 'dplyr/remote_task'
|
4
|
+
|
5
|
+
module Dplyr
|
6
|
+
class TaskRunner
|
7
|
+
|
8
|
+
include ::Dply::Logger
|
9
|
+
|
10
|
+
attr_reader :hosts, :parallel_jobs, :task, :messages, :exit_statuses
|
11
|
+
attr_writer :auto_serialize
|
12
|
+
|
13
|
+
def initialize(hosts, task, parallel_jobs: 1)
|
14
|
+
@hosts = hosts
|
15
|
+
@parallel_jobs = parallel_jobs
|
16
|
+
@task = task
|
17
|
+
|
18
|
+
@messages = {}
|
19
|
+
@exit_statuses = {}
|
20
|
+
@auto_serialize = true
|
21
|
+
end
|
22
|
+
|
23
|
+
def run
|
24
|
+
if parallel_jobs > 1 && hosts.count > 1
|
25
|
+
run_in_parallel
|
26
|
+
else
|
27
|
+
run_serially
|
28
|
+
end
|
29
|
+
report.print_full
|
30
|
+
end
|
31
|
+
|
32
|
+
def run_serially
|
33
|
+
hosts.each do |host|
|
34
|
+
task = execute_serially host
|
35
|
+
break if task.exit_status != 0
|
36
|
+
end
|
37
|
+
end
|
38
|
+
|
39
|
+
def run_in_parallel
|
40
|
+
if @auto_serialize
|
41
|
+
execute_serially hosts[0]
|
42
|
+
execute_in_parallel Range.new(1,hosts.size - 2)
|
43
|
+
execute_serially hosts[-1]
|
44
|
+
else
|
45
|
+
execute_in_parallel Range.new(0, hosts.size - 1)
|
46
|
+
end
|
47
|
+
end
|
48
|
+
|
49
|
+
private
|
50
|
+
|
51
|
+
def remote_task(host)
|
52
|
+
RemoteTask.new(host, @task, id_size: host_id_max_width)
|
53
|
+
end
|
54
|
+
|
55
|
+
def execute_serially(host)
|
56
|
+
task = remote_task host
|
57
|
+
task.run
|
58
|
+
@exit_statuses[host] = task.exit_status
|
59
|
+
@messages[host] = task.messages
|
60
|
+
return task
|
61
|
+
end
|
62
|
+
|
63
|
+
def execute_in_parallel(range)
|
64
|
+
init_run
|
65
|
+
queue_hosts range
|
66
|
+
parallel_jobs.times do
|
67
|
+
spawn_queued_job
|
68
|
+
end
|
69
|
+
loop do
|
70
|
+
break if @queue.empty?
|
71
|
+
@mq.pop
|
72
|
+
spawn_queued_job
|
73
|
+
end
|
74
|
+
wait_for_threads
|
75
|
+
end
|
76
|
+
|
77
|
+
def start_task_thread(host)
|
78
|
+
t = Thread.new do
|
79
|
+
Thread.current[:messages] = []
|
80
|
+
task = remote_task(host)
|
81
|
+
begin
|
82
|
+
task.run
|
83
|
+
Thread.current[:messages] = task.messages
|
84
|
+
Thread.current[:exit_status] = task.exit_status
|
85
|
+
rescue => e
|
86
|
+
puts e.message
|
87
|
+
ensure
|
88
|
+
@mq << true
|
89
|
+
end
|
90
|
+
end
|
91
|
+
t.abort_on_exception = true
|
92
|
+
t.run
|
93
|
+
@threads[host] = t
|
94
|
+
end
|
95
|
+
|
96
|
+
def init_run
|
97
|
+
@mq = Queue.new
|
98
|
+
@queue = Queue.new
|
99
|
+
@threads = {}
|
100
|
+
end
|
101
|
+
|
102
|
+
def queue_hosts(range)
|
103
|
+
range.each { |i| @queue << hosts[i] }
|
104
|
+
end
|
105
|
+
|
106
|
+
def spawn_queued_job
|
107
|
+
return if @queue.empty?
|
108
|
+
host = @queue.pop(false)
|
109
|
+
start_task_thread host
|
110
|
+
rescue ThreadError
|
111
|
+
end
|
112
|
+
|
113
|
+
def host_id_max_width
|
114
|
+
@host_id_max_width ||= hosts.map {|h| h[:id].size }.max
|
115
|
+
end
|
116
|
+
|
117
|
+
def report
|
118
|
+
@report ||= Report.new(hosts, exit_statuses, messages)
|
119
|
+
end
|
120
|
+
|
121
|
+
def wait_for_threads
|
122
|
+
@threads.each do |host, t|
|
123
|
+
@exit_statuses[host] = t.value
|
124
|
+
@messages[host] = t[:messages]
|
125
|
+
end
|
126
|
+
end
|
127
|
+
|
128
|
+
end
|
129
|
+
end
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: dply
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.1.
|
4
|
+
version: 0.1.17
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Neeraj
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2015-
|
11
|
+
date: 2015-05-03 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: ruby-elf
|
@@ -98,9 +98,11 @@ files:
|
|
98
98
|
- lib/dply/cli/build.rb
|
99
99
|
- lib/dply/cli/depcheck.rb
|
100
100
|
- lib/dply/cli/deploy.rb
|
101
|
+
- lib/dply/cli/devbuild.rb
|
101
102
|
- lib/dply/cli/install_pkgs.rb
|
102
103
|
- lib/dply/cli/reload.rb
|
103
104
|
- lib/dply/cli/reopen_logs.rb
|
105
|
+
- lib/dply/cli/setup.rb
|
104
106
|
- lib/dply/cli/status.rb
|
105
107
|
- lib/dply/cli/task.rb
|
106
108
|
- lib/dply/config.rb
|
@@ -128,6 +130,9 @@ files:
|
|
128
130
|
- lib/dply/strategy/archive.rb
|
129
131
|
- lib/dply/strategy/git.rb
|
130
132
|
- lib/dply/tasks.rb
|
133
|
+
- lib/dply/templates/build.erb
|
134
|
+
- lib/dply/templates/deploy.erb
|
135
|
+
- lib/dply/templates/pkgs.erb
|
131
136
|
- lib/dply/version.rb
|
132
137
|
- lib/dply/yum.rb
|
133
138
|
- lib/dplyr/cli.rb
|
@@ -135,6 +140,7 @@ files:
|
|
135
140
|
- lib/dplyr/report.rb
|
136
141
|
- lib/dplyr/stage.rb
|
137
142
|
- lib/dplyr/stages_config.rb
|
143
|
+
- lib/dplyr/task_runner.rb
|
138
144
|
- lib/dplyr/tasks_config.rb
|
139
145
|
homepage: ''
|
140
146
|
licenses:
|
@@ -156,7 +162,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
156
162
|
version: '0'
|
157
163
|
requirements: []
|
158
164
|
rubyforge_project:
|
159
|
-
rubygems_version: 2.
|
165
|
+
rubygems_version: 2.4.5
|
160
166
|
signing_key:
|
161
167
|
specification_version: 4
|
162
168
|
summary: rake based deploy tool
|