dply 0.1.13 → 0.1.17
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/bin/dplyr +9 -1
- data/bin/drake +3 -1
- data/docs/stages.rb +9 -4
- data/lib/dply/cli/devbuild.rb +67 -0
- data/lib/dply/cli/setup.rb +102 -0
- data/lib/dply/config.rb +4 -0
- data/lib/dply/config_struct.rb +1 -1
- data/lib/dply/custom_logger.rb +6 -1
- data/lib/dply/git.rb +6 -1
- data/lib/dply/pkgs_config.rb +1 -1
- data/lib/dply/repo.rb +1 -1
- data/lib/dply/setup.rb +1 -1
- data/lib/dply/tasks.rb +29 -5
- data/lib/dply/templates/build.erb +5 -0
- data/lib/dply/templates/deploy.erb +15 -0
- data/lib/dply/templates/pkgs.erb +5 -0
- data/lib/dply/version.rb +1 -1
- data/lib/dply/yum.rb +4 -4
- data/lib/dplyr/cli.rb +3 -3
- data/lib/dplyr/remote_task.rb +45 -111
- data/lib/dplyr/report.rb +4 -1
- data/lib/dplyr/stage.rb +24 -3
- data/lib/dplyr/task_runner.rb +129 -0
- metadata +9 -3
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: e36e026a2a88ee74042f732298e42fb7f13a4eca
|
4
|
+
data.tar.gz: b82102df075f09482830a6e48850710a6788b09a
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: e0d0e00e407aab69f5394aad3dc2e8833c41db1c2869eff653f5dd396cb681f1fd017e8211bf7795c9f1ce9001fba5b5f8885b113844bf7eae9d66b89ec999e8
|
7
|
+
data.tar.gz: 27983c97231765d2b2387c02e1f234fd8b060fcd2652389047be8cbeeb05cbd1e19e98380b8de502d2db0161740067a2dc3ef691f6733c2972dcf0367f495991
|
data/bin/dplyr
CHANGED
@@ -45,10 +45,14 @@ opts_parser = OptionParser.new do |opts|
|
|
45
45
|
logger.level = ::Logger::DEBUG
|
46
46
|
end
|
47
47
|
|
48
|
+
opts.on("-m", "--enable-markers", "Enable marker logging") do
|
49
|
+
logger.enable_markers = true
|
50
|
+
end
|
51
|
+
|
48
52
|
opts.on_tail("-h", "--help", "Show this message") do
|
49
53
|
puts opts
|
50
54
|
exit
|
51
|
-
end
|
55
|
+
end
|
52
56
|
end
|
53
57
|
begin
|
54
58
|
opts_parser.order!(ARGV)
|
@@ -70,4 +74,8 @@ rescue OptionParser::InvalidOption, OptionParser::MissingArgument, ::Dply::Error
|
|
70
74
|
logger.error "#{e.message}"
|
71
75
|
abort
|
72
76
|
end
|
77
|
+
ensure
|
78
|
+
logger.marker "end"
|
73
79
|
end
|
80
|
+
|
81
|
+
|
data/bin/drake
CHANGED
@@ -71,7 +71,9 @@ begin
|
|
71
71
|
|
72
72
|
command = (ARGV.shift || "").to_sym
|
73
73
|
case command
|
74
|
-
when :deploy, :reload, :task,
|
74
|
+
when :deploy, :reload, :task,
|
75
|
+
:build, :depcheck, :"install-pkgs", :reopen_logs,
|
76
|
+
:status, :devbuild, :setup
|
75
77
|
run_cli command, ARGV
|
76
78
|
when /\A[a-zA-Z_\-0-9]+[:][a-zA-Z_\-0-9]+\z/
|
77
79
|
require 'dply/tasks'
|
data/docs/stages.rb
CHANGED
@@ -1,8 +1,13 @@
|
|
1
|
+
# Sample stages.rb
|
2
|
+
# __________________________________________________________
|
3
|
+
#
|
4
|
+
|
1
5
|
stage :production do
|
2
6
|
user "dev"
|
3
|
-
|
4
|
-
deploy_dir "/home/dev/project"
|
7
|
+
dir "/home/dev/project"
|
5
8
|
parallel_runs 1
|
6
|
-
host "10.1.1.1",
|
7
|
-
host "10.1.1.2",
|
9
|
+
host "10.1.1.1", dir: "/home/dev/project1", id: "project1", user: "dev1" ## default roles: ["first"]
|
10
|
+
host "10.1.1.2", dir: "/home/dev/project2", id: "project2"
|
11
|
+
host "10.1.1.3", roles: ["db"]
|
12
|
+
host "10.1.1.4" ## default roles: ["last"]
|
8
13
|
end
|
@@ -0,0 +1,67 @@
|
|
1
|
+
require 'dply/logger'
|
2
|
+
require 'dply/lock'
|
3
|
+
require 'dply/tasks'
|
4
|
+
require 'fileutils'
|
5
|
+
|
6
|
+
module Dply
|
7
|
+
module Cli
|
8
|
+
class Devbuild
|
9
|
+
|
10
|
+
include Logger
|
11
|
+
|
12
|
+
def initialize(argv)
|
13
|
+
@argv = argv
|
14
|
+
@options = {}
|
15
|
+
end
|
16
|
+
|
17
|
+
def run
|
18
|
+
lock.acquire
|
19
|
+
opts.parse!(@argv)
|
20
|
+
revision = @options[:revision] || "dev"
|
21
|
+
ENV["BUILD_NUMBER"] = revision
|
22
|
+
|
23
|
+
build_artifacts = "tmp/build_artifacts"
|
24
|
+
FileUtils.mkdir_p build_artifacts
|
25
|
+
|
26
|
+
clear_bundle_config
|
27
|
+
tasks.install_pkgs(build_mode: true, use_yum: @options[:use_yum])
|
28
|
+
tasks.build "build:default"
|
29
|
+
ensure
|
30
|
+
clear_bundle_config
|
31
|
+
end
|
32
|
+
|
33
|
+
def tasks
|
34
|
+
@tasks ||= ::Dply::Tasks.new
|
35
|
+
end
|
36
|
+
|
37
|
+
def lock
|
38
|
+
@lock ||= ::Dply::Lock.new
|
39
|
+
end
|
40
|
+
|
41
|
+
def clear_bundle_config
|
42
|
+
FileUtils.rm ".bundle/config" if File.exists? ".bundle/config"
|
43
|
+
end
|
44
|
+
|
45
|
+
def opts
|
46
|
+
OptionParser.new do |opts|
|
47
|
+
|
48
|
+
opts.banner = "Usage: drake devbuild [options] [target]"
|
49
|
+
|
50
|
+
opts.on("-r", "--revision [REVISION]", "Specify revision") do |r|
|
51
|
+
@options[:revision] = r
|
52
|
+
end
|
53
|
+
|
54
|
+
opts.on("--use-yum", "use yum to install packages") do |e|
|
55
|
+
@options[:use_yum] = true
|
56
|
+
end
|
57
|
+
|
58
|
+
opts.on("-h", "--help", "Help") do
|
59
|
+
puts opts
|
60
|
+
exit
|
61
|
+
end
|
62
|
+
end
|
63
|
+
end
|
64
|
+
|
65
|
+
end
|
66
|
+
end
|
67
|
+
end
|
@@ -0,0 +1,102 @@
|
|
1
|
+
require 'dply/logger'
|
2
|
+
require 'dply/lock'
|
3
|
+
require 'fileutils'
|
4
|
+
require 'erb'
|
5
|
+
|
6
|
+
module Dply
|
7
|
+
module Cli
|
8
|
+
class Setup
|
9
|
+
|
10
|
+
include ::Dply::Logger
|
11
|
+
|
12
|
+
def initialize(argv)
|
13
|
+
@argv = argv
|
14
|
+
@options = {}
|
15
|
+
end
|
16
|
+
|
17
|
+
def run
|
18
|
+
opts.parse!(@argv)
|
19
|
+
return if not proceed?
|
20
|
+
lock.acquire
|
21
|
+
setup_default
|
22
|
+
namespace = @argv.shift
|
23
|
+
if namespace
|
24
|
+
setup_task(namespace)
|
25
|
+
else
|
26
|
+
setup_build_task
|
27
|
+
setup_task(:production)
|
28
|
+
end
|
29
|
+
end
|
30
|
+
|
31
|
+
def proceed?
|
32
|
+
print "Are you sure?(y/n) "
|
33
|
+
v = STDIN.gets.strip
|
34
|
+
v == "y"
|
35
|
+
end
|
36
|
+
|
37
|
+
def setup_default
|
38
|
+
if not File.exist? "dply"
|
39
|
+
FileUtils.mkdir_p "dply"
|
40
|
+
logger.info "created dply/ dir"
|
41
|
+
else
|
42
|
+
logger.info "skipping dply/ dir"
|
43
|
+
end
|
44
|
+
|
45
|
+
rakefile = "dply/Rakefile"
|
46
|
+
if not File.exist? rakefile
|
47
|
+
FileUtils.touch rakefile
|
48
|
+
logger.info "created #{rakefile}"
|
49
|
+
else
|
50
|
+
logger.info "skipping #{rakefile}"
|
51
|
+
end
|
52
|
+
|
53
|
+
pkgs_yml = "pkgs.yml"
|
54
|
+
if not File.exist? pkgs_yml
|
55
|
+
FileUtils.cp "#{templates_dir}/pkgs.erb", pkgs_yml
|
56
|
+
logger.info "created #{pkgs_yml}"
|
57
|
+
else
|
58
|
+
logger.info "skipping #{pkgs_yml}"
|
59
|
+
end
|
60
|
+
end
|
61
|
+
|
62
|
+
def setup_task(namespace, template: "deploy.erb")
|
63
|
+
tasks_file = "dply/#{namespace}.rake"
|
64
|
+
if File.exist? tasks_file
|
65
|
+
logger.info "skipping #{tasks_file}"
|
66
|
+
return
|
67
|
+
end
|
68
|
+
b = binding
|
69
|
+
template_path = "#{templates_dir}/#{template}"
|
70
|
+
erb = ERB.new(File.read(template_path), nil, '-')
|
71
|
+
File.open(tasks_file, 'w') { |f| f.write erb.result(b) }
|
72
|
+
logger.info "created #{tasks_file}"
|
73
|
+
end
|
74
|
+
|
75
|
+
def setup_build_task
|
76
|
+
setup_task("build", template: "build.erb")
|
77
|
+
end
|
78
|
+
|
79
|
+
def templates_dir
|
80
|
+
@templates_dir ||= "#{__dir__}/../templates"
|
81
|
+
end
|
82
|
+
|
83
|
+
def lock
|
84
|
+
@lock ||= ::Dply::Lock.new
|
85
|
+
end
|
86
|
+
|
87
|
+
def opts
|
88
|
+
OptionParser.new do |opts|
|
89
|
+
|
90
|
+
opts.banner = "Usage: drake setup [options] [namespace]"
|
91
|
+
|
92
|
+
opts.on("-h", "--help", "Help") do
|
93
|
+
puts opts
|
94
|
+
exit
|
95
|
+
end
|
96
|
+
end
|
97
|
+
end
|
98
|
+
|
99
|
+
|
100
|
+
end
|
101
|
+
end
|
102
|
+
end
|
data/lib/dply/config.rb
CHANGED
data/lib/dply/config_struct.rb
CHANGED
@@ -4,7 +4,7 @@ module Dply
|
|
4
4
|
|
5
5
|
attr_writer :revision, :build_url, :build_url_proc,
|
6
6
|
:revision_proc
|
7
|
-
attr_accessor :dir, :name, :repo, :branch,
|
7
|
+
attr_accessor :dir, :name, :repo, :branch, :mirror,
|
8
8
|
:strategy, :target, :verify_checksum,
|
9
9
|
:config_map, :dir_map, :shared_dirs,
|
10
10
|
:config_download_url, :config_skip_download
|
data/lib/dply/custom_logger.rb
CHANGED
@@ -3,7 +3,7 @@ require 'dply/ext/string'
|
|
3
3
|
module Dply
|
4
4
|
class CustomLogger < ::Logger
|
5
5
|
|
6
|
-
attr_writer :trace_mode, :remote_mode
|
6
|
+
attr_writer :trace_mode, :remote_mode, :enable_markers
|
7
7
|
|
8
8
|
def initialize(file)
|
9
9
|
super(file)
|
@@ -37,5 +37,10 @@ module Dply
|
|
37
37
|
puts %{dply_msg|#{msg}}
|
38
38
|
end
|
39
39
|
|
40
|
+
def marker(msg)
|
41
|
+
return if not @enable_markers
|
42
|
+
puts "dply_marker:#{msg}"
|
43
|
+
end
|
44
|
+
|
40
45
|
end
|
41
46
|
end
|
data/lib/dply/git.rb
CHANGED
@@ -20,7 +20,12 @@ module Dply
|
|
20
20
|
end
|
21
21
|
|
22
22
|
def self.clone(repo, dir, mirror: nil)
|
23
|
-
|
23
|
+
if mirror
|
24
|
+
cmd "git clone #{mirror} #{dir}"
|
25
|
+
Dir.chdir(dir) { cmd "git remote set-url origin #{repo}" }
|
26
|
+
else
|
27
|
+
cmd "git clone #{repo} #{dir}"
|
28
|
+
end
|
24
29
|
end
|
25
30
|
|
26
31
|
def self.clean
|
data/lib/dply/pkgs_config.rb
CHANGED
data/lib/dply/repo.rb
CHANGED
data/lib/dply/setup.rb
CHANGED
data/lib/dply/tasks.rb
CHANGED
@@ -4,6 +4,8 @@ require 'dply/bundle'
|
|
4
4
|
require 'dply/yum'
|
5
5
|
require 'dply/linker'
|
6
6
|
require 'dply/pkgs_config'
|
7
|
+
require 'dply/error'
|
8
|
+
require 'etc'
|
7
9
|
|
8
10
|
module Dply
|
9
11
|
class Tasks
|
@@ -49,9 +51,12 @@ module Dply
|
|
49
51
|
def install_pkgs(build_mode: false, use_yum: false)
|
50
52
|
return if not File.exists? "pkgs.yml"
|
51
53
|
drake_exists = File.exists? (drake_command)
|
54
|
+
|
55
|
+
pkgs = get_pkgs(build_mode)
|
52
56
|
if use_yum || !drake_exists
|
53
|
-
yum_install
|
57
|
+
yum_install pkgs
|
54
58
|
else
|
59
|
+
return if Yum.new(pkgs).installed?
|
55
60
|
command_install build_mode
|
56
61
|
end
|
57
62
|
end
|
@@ -70,20 +75,39 @@ module Dply
|
|
70
75
|
@bundle ||= Bundle.new(deployment: @deployment)
|
71
76
|
end
|
72
77
|
|
73
|
-
def
|
74
|
-
|
78
|
+
def get_pkgs(build_mode)
|
79
|
+
PkgsConfig.new(build_mode: build_mode).pkgs
|
80
|
+
end
|
81
|
+
|
82
|
+
def yum_install(pkgs)
|
75
83
|
Yum.new(pkgs, sudo: true).install
|
76
84
|
end
|
77
85
|
|
78
86
|
def command_install(build_mode)
|
79
|
-
command = "
|
87
|
+
command = "#{drake_command} install-pkgs"
|
80
88
|
command << " -b" if build_mode
|
81
|
-
|
89
|
+
check_sudo_permission command
|
90
|
+
cmd "sudo -n #{command}"
|
82
91
|
end
|
83
92
|
|
84
93
|
def drake_command
|
85
94
|
@drake_command ||= (ENV["DRAKE_COMMAND"] || "/opt/ruby/bin/drake")
|
86
95
|
end
|
87
96
|
|
97
|
+
def check_sudo_permission(command)
|
98
|
+
output = `sudo -n -l #{command}`
|
99
|
+
if output.chomp.strip == command
|
100
|
+
return true
|
101
|
+
else
|
102
|
+
msg = []
|
103
|
+
user = Etc.getpwuid(Process.uid).name
|
104
|
+
msg << %{unable to run "#{command}" with sudo permissions}
|
105
|
+
msg << %{To resolve add the following line to sudoers: }
|
106
|
+
msg << %{#{user} ALL=(ALL) NOPASSWD: /opt/ruby/bin/drake install-pkgs *, /opt/ruby/bin/drake install-pkgs}.yellow
|
107
|
+
raise Error, msg.join("\n")
|
108
|
+
end
|
109
|
+
|
110
|
+
end
|
111
|
+
|
88
112
|
end
|
89
113
|
end
|
data/lib/dply/version.rb
CHANGED
data/lib/dply/yum.rb
CHANGED
@@ -26,6 +26,10 @@ module Dply
|
|
26
26
|
end
|
27
27
|
end
|
28
28
|
|
29
|
+
def installed?
|
30
|
+
not_installed_pkgs.size == 0
|
31
|
+
end
|
32
|
+
|
29
33
|
private
|
30
34
|
|
31
35
|
def pkgs_str
|
@@ -42,9 +46,5 @@ module Dply
|
|
42
46
|
matches = `#{command}`.scan(/^package (.*) is not installed$/)
|
43
47
|
end
|
44
48
|
|
45
|
-
def installed?
|
46
|
-
not_installed_pkgs.size == 0
|
47
|
-
end
|
48
|
-
|
49
49
|
end
|
50
50
|
end
|
data/lib/dplyr/cli.rb
CHANGED
@@ -1,5 +1,5 @@
|
|
1
1
|
require 'dplyr/stages_config'
|
2
|
-
require 'dplyr/
|
2
|
+
require 'dplyr/task_runner'
|
3
3
|
require 'dply/logger'
|
4
4
|
|
5
5
|
module Dplyr
|
@@ -52,8 +52,8 @@ module Dplyr
|
|
52
52
|
end
|
53
53
|
|
54
54
|
def run_remote_task
|
55
|
-
|
56
|
-
|
55
|
+
task_runner = TaskRunner.new(hosts, argv_str, parallel_jobs: parallel_jobs)
|
56
|
+
task_runner.run
|
57
57
|
end
|
58
58
|
|
59
59
|
end
|
data/lib/dplyr/remote_task.rb
CHANGED
@@ -6,85 +6,49 @@ module Dplyr
|
|
6
6
|
class RemoteTask
|
7
7
|
|
8
8
|
include ::Dply::Logger
|
9
|
-
|
10
|
-
attr_reader :hosts, :parallel_jobs, :task, :env
|
11
9
|
|
12
|
-
|
13
|
-
|
14
|
-
|
10
|
+
attr_reader :exit_status, :messages
|
11
|
+
|
12
|
+
def initialize(host, task, id_size: nil)
|
13
|
+
@host = host
|
15
14
|
@task = task
|
16
|
-
@
|
17
|
-
@
|
15
|
+
@messages = []
|
16
|
+
@id_size = id_size || @host_info[:id].size
|
18
17
|
end
|
19
18
|
|
20
19
|
def run
|
21
|
-
|
22
|
-
|
23
|
-
|
24
|
-
|
25
|
-
end
|
20
|
+
reset!
|
21
|
+
r, w, pid = PTY.spawn(remote_cmd)
|
22
|
+
pty_read(r, @host[:id])
|
23
|
+
@exit_status = get_exit_status pid
|
26
24
|
end
|
27
25
|
|
28
|
-
def
|
29
|
-
|
30
|
-
|
31
|
-
|
32
|
-
|
33
|
-
loop do
|
34
|
-
break if @queue.empty?
|
35
|
-
@mq.pop
|
36
|
-
spawn_queued_job
|
37
|
-
end
|
38
|
-
report.print_full
|
39
|
-
end
|
40
|
-
|
41
|
-
def run_serially
|
42
|
-
hosts.each do |host_info|
|
43
|
-
puts "=== Running on #{host_info[:id]} ==="
|
44
|
-
run_cmd = remote_cmd host_info
|
45
|
-
system run_cmd
|
46
|
-
puts
|
47
|
-
raise ::Dply::Error, "remote deploy failed on #{host_info[:id]}" if $? != 0
|
26
|
+
def remote_cmd
|
27
|
+
if logger.debug?
|
28
|
+
%(#{ssh} -l #{user} #{addr} '#{env} drake --remote --debug -d #{dir} #{@task} 2>&1')
|
29
|
+
else
|
30
|
+
%(#{ssh} -l #{user} #{addr} '#{env} drake --remote -d #{dir} #{@task} 2>&1' 2>/dev/null)
|
48
31
|
end
|
49
32
|
end
|
50
33
|
|
51
|
-
|
52
|
-
|
53
|
-
def init_run
|
54
|
-
queue_all_hosts
|
55
|
-
job_output_template
|
56
|
-
@threads = {}
|
57
|
-
@mq = Queue.new
|
34
|
+
def user
|
35
|
+
@host.fetch :user
|
58
36
|
end
|
59
37
|
|
60
|
-
def
|
61
|
-
@
|
62
|
-
hosts.each { |h| @queue << h }
|
38
|
+
def dir
|
39
|
+
@host.fetch :dir
|
63
40
|
end
|
64
41
|
|
65
|
-
def
|
66
|
-
|
67
|
-
thread = popen(run_cmd, host_info)
|
68
|
-
@threads[host_info] = thread
|
42
|
+
def addr
|
43
|
+
@host.fetch :addr
|
69
44
|
end
|
70
45
|
|
71
|
-
def
|
72
|
-
|
73
|
-
host = @queue.pop(false)
|
74
|
-
spawn_job host
|
75
|
-
rescue ThreadError
|
46
|
+
def roles
|
47
|
+
@host.fetch :roles
|
76
48
|
end
|
77
|
-
|
78
|
-
def
|
79
|
-
|
80
|
-
host = host_info[:host]
|
81
|
-
dir = host_info[:dir]
|
82
|
-
ssh = "ssh -tt -o BatchMode=yes -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null"
|
83
|
-
if logger.debug?
|
84
|
-
%(#{ssh} -l #{user} #{host} '#{env} drake --remote --debug -d #{dir} #{task} 2>&1')
|
85
|
-
else
|
86
|
-
%(#{ssh} -l #{user} #{host} '#{env} drake --remote -d #{dir} #{task} 2>&1' 2>/dev/null)
|
87
|
-
end
|
49
|
+
|
50
|
+
def ssh
|
51
|
+
"ssh -tt -o BatchMode=yes -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null"
|
88
52
|
end
|
89
53
|
|
90
54
|
def pty_read(file, id)
|
@@ -92,72 +56,42 @@ module Dplyr
|
|
92
56
|
if line =~ /\Adply_msg\|/
|
93
57
|
receive_message line
|
94
58
|
else
|
95
|
-
printf
|
59
|
+
printf output_template, id, line
|
96
60
|
end
|
97
61
|
end
|
98
62
|
rescue EOFError,Errno::ECONNRESET, Errno::EPIPE, Errno::EIO => e
|
99
63
|
end
|
100
64
|
|
101
|
-
def popen(cmd, host_info)
|
102
|
-
t = Thread.new do |t|
|
103
|
-
Thread.current[:messages] = []
|
104
|
-
begin
|
105
|
-
r, w, pid = PTY.spawn(cmd)
|
106
|
-
pty_read(r, host_info[:id])
|
107
|
-
exit_status pid
|
108
|
-
ensure
|
109
|
-
@mq << true
|
110
|
-
end
|
111
|
-
end
|
112
|
-
t.abort_on_exception = true
|
113
|
-
t.run
|
114
|
-
return t
|
115
|
-
end
|
116
|
-
|
117
|
-
def host_id_max_width
|
118
|
-
hosts.map {|h| h[:id].size }.max
|
119
|
-
end
|
120
|
-
|
121
|
-
def job_output_template
|
122
|
-
@job_output_template ||= begin
|
123
|
-
id_template = "%-#{host_id_max_width}s".bold.grey
|
124
|
-
template = "#{id_template} %s"
|
125
|
-
end
|
126
|
-
end
|
127
|
-
|
128
65
|
def receive_message(msg_str)
|
129
66
|
msg = msg_str.partition("|")[2].strip
|
130
67
|
return if msg.empty?
|
131
|
-
messages
|
132
|
-
messages << msg
|
68
|
+
@messages << msg
|
133
69
|
end
|
134
70
|
|
135
|
-
def
|
136
|
-
|
71
|
+
def get_exit_status(pid)
|
72
|
+
pid, status = Process.waitpid2(pid)
|
73
|
+
return status
|
137
74
|
end
|
138
75
|
|
139
|
-
def
|
140
|
-
@
|
141
|
-
|
142
|
-
|
143
|
-
m[host] = thread[:messages]
|
144
|
-
end
|
145
|
-
m
|
76
|
+
def output_template
|
77
|
+
@output_template ||= begin
|
78
|
+
id_template = "%-#{@id_size}s".bold.grey
|
79
|
+
template = "#{id_template} %s"
|
146
80
|
end
|
147
81
|
end
|
148
82
|
|
149
|
-
def
|
150
|
-
|
151
|
-
@exit_statuses = {}
|
152
|
-
@threads.each do |host, thread|
|
153
|
-
@exit_statuses[host] = thread.value
|
154
|
-
end
|
155
|
-
@exit_statuses
|
83
|
+
def reset!
|
84
|
+
@output_template = nil
|
156
85
|
end
|
157
86
|
|
158
|
-
def
|
159
|
-
|
160
|
-
|
87
|
+
def env
|
88
|
+
@env ||= "PATH=/usr/sbin:/usr/local/sbin:$PATH #{roles_env}"
|
89
|
+
end
|
90
|
+
|
91
|
+
def roles_env
|
92
|
+
return "" if not roles.size > 0
|
93
|
+
roles_str = roles.join(",")
|
94
|
+
"DPLY_ROLES=#{roles_str}"
|
161
95
|
end
|
162
96
|
|
163
97
|
end
|
data/lib/dplyr/report.rb
CHANGED
@@ -1,6 +1,8 @@
|
|
1
|
+
require 'dply/logger'
|
1
2
|
module Dplyr
|
2
3
|
class Report
|
3
4
|
|
5
|
+
include ::Dply::Logger
|
4
6
|
attr_reader :hosts, :exit_statuses
|
5
7
|
|
6
8
|
def initialize(hosts, exit_statuses, messages)
|
@@ -11,6 +13,7 @@ module Dplyr
|
|
11
13
|
|
12
14
|
|
13
15
|
def print_full
|
16
|
+
logger.marker "summary_start"
|
14
17
|
print_successful_jobs
|
15
18
|
print_failed_jobs
|
16
19
|
print_summary
|
@@ -52,7 +55,7 @@ module Dplyr
|
|
52
55
|
run_count = @exit_statuses.count
|
53
56
|
not_run = total_hosts - run_count
|
54
57
|
if (failed.count > 0 || not_run > 0 )
|
55
|
-
not_run_error = "not run on #{not_run}
|
58
|
+
not_run_error = "not run on #{not_run} of #{total_hosts} hosts" if not_run > 0
|
56
59
|
failed_error = "failed on #{failed.count} of #{total_hosts} hosts" if failed.count > 0
|
57
60
|
|
58
61
|
errors = []
|
data/lib/dplyr/stage.rb
CHANGED
@@ -1,6 +1,10 @@
|
|
1
|
+
require 'dply/helper'
|
2
|
+
|
1
3
|
module Dplyr
|
2
4
|
class Stage
|
3
5
|
|
6
|
+
include ::Dply::Helper
|
7
|
+
|
4
8
|
attr_accessor :config_proc
|
5
9
|
|
6
10
|
def initialize(name)
|
@@ -18,12 +22,13 @@ module Dplyr
|
|
18
22
|
})
|
19
23
|
end
|
20
24
|
|
21
|
-
def host(
|
25
|
+
def host(addr, user: nil, dir: nil, id: nil, roles: [])
|
22
26
|
@hosts << ({
|
23
|
-
|
27
|
+
addr: addr,
|
24
28
|
user: user,
|
25
29
|
dir: dir,
|
26
|
-
id: id ||
|
30
|
+
id: id || addr,
|
31
|
+
roles: cleaned_roles(roles)
|
27
32
|
})
|
28
33
|
end
|
29
34
|
|
@@ -54,15 +59,31 @@ module Dplyr
|
|
54
59
|
return value
|
55
60
|
end
|
56
61
|
|
62
|
+
def add_default_roles
|
63
|
+
return if not @hosts.size > 0
|
64
|
+
@hosts.first[:roles] << "first"
|
65
|
+
@hosts.last[:roles] << "last"
|
66
|
+
end
|
67
|
+
|
57
68
|
def finalize
|
58
69
|
return if @finalized
|
59
70
|
instance_eval &config_proc if config_proc
|
60
71
|
fill_hosts
|
72
|
+
add_default_roles
|
61
73
|
@hosts.freeze
|
62
74
|
@parallel_runs.freeze
|
63
75
|
@finalized = true
|
64
76
|
end
|
65
77
|
|
78
|
+
def cleaned_roles(roles)
|
79
|
+
error "roles must be an array" if not roles.is_a? Array
|
80
|
+
roles.each do |r|
|
81
|
+
r.strip!
|
82
|
+
error "invalid role value #{r} " if not r =~ /\A[0-9A-Za-z_\-]+\z/
|
83
|
+
end
|
84
|
+
return roles
|
85
|
+
end
|
86
|
+
|
66
87
|
end
|
67
88
|
end
|
68
89
|
|
@@ -0,0 +1,129 @@
|
|
1
|
+
require 'dplyr/report'
|
2
|
+
require 'dply/logger'
|
3
|
+
require 'dplyr/remote_task'
|
4
|
+
|
5
|
+
module Dplyr
|
6
|
+
class TaskRunner
|
7
|
+
|
8
|
+
include ::Dply::Logger
|
9
|
+
|
10
|
+
attr_reader :hosts, :parallel_jobs, :task, :messages, :exit_statuses
|
11
|
+
attr_writer :auto_serialize
|
12
|
+
|
13
|
+
def initialize(hosts, task, parallel_jobs: 1)
|
14
|
+
@hosts = hosts
|
15
|
+
@parallel_jobs = parallel_jobs
|
16
|
+
@task = task
|
17
|
+
|
18
|
+
@messages = {}
|
19
|
+
@exit_statuses = {}
|
20
|
+
@auto_serialize = true
|
21
|
+
end
|
22
|
+
|
23
|
+
def run
|
24
|
+
if parallel_jobs > 1 && hosts.count > 1
|
25
|
+
run_in_parallel
|
26
|
+
else
|
27
|
+
run_serially
|
28
|
+
end
|
29
|
+
report.print_full
|
30
|
+
end
|
31
|
+
|
32
|
+
def run_serially
|
33
|
+
hosts.each do |host|
|
34
|
+
task = execute_serially host
|
35
|
+
break if task.exit_status != 0
|
36
|
+
end
|
37
|
+
end
|
38
|
+
|
39
|
+
def run_in_parallel
|
40
|
+
if @auto_serialize
|
41
|
+
execute_serially hosts[0]
|
42
|
+
execute_in_parallel Range.new(1,hosts.size - 2)
|
43
|
+
execute_serially hosts[-1]
|
44
|
+
else
|
45
|
+
execute_in_parallel Range.new(0, hosts.size - 1)
|
46
|
+
end
|
47
|
+
end
|
48
|
+
|
49
|
+
private
|
50
|
+
|
51
|
+
def remote_task(host)
|
52
|
+
RemoteTask.new(host, @task, id_size: host_id_max_width)
|
53
|
+
end
|
54
|
+
|
55
|
+
def execute_serially(host)
|
56
|
+
task = remote_task host
|
57
|
+
task.run
|
58
|
+
@exit_statuses[host] = task.exit_status
|
59
|
+
@messages[host] = task.messages
|
60
|
+
return task
|
61
|
+
end
|
62
|
+
|
63
|
+
def execute_in_parallel(range)
|
64
|
+
init_run
|
65
|
+
queue_hosts range
|
66
|
+
parallel_jobs.times do
|
67
|
+
spawn_queued_job
|
68
|
+
end
|
69
|
+
loop do
|
70
|
+
break if @queue.empty?
|
71
|
+
@mq.pop
|
72
|
+
spawn_queued_job
|
73
|
+
end
|
74
|
+
wait_for_threads
|
75
|
+
end
|
76
|
+
|
77
|
+
def start_task_thread(host)
|
78
|
+
t = Thread.new do
|
79
|
+
Thread.current[:messages] = []
|
80
|
+
task = remote_task(host)
|
81
|
+
begin
|
82
|
+
task.run
|
83
|
+
Thread.current[:messages] = task.messages
|
84
|
+
Thread.current[:exit_status] = task.exit_status
|
85
|
+
rescue => e
|
86
|
+
puts e.message
|
87
|
+
ensure
|
88
|
+
@mq << true
|
89
|
+
end
|
90
|
+
end
|
91
|
+
t.abort_on_exception = true
|
92
|
+
t.run
|
93
|
+
@threads[host] = t
|
94
|
+
end
|
95
|
+
|
96
|
+
def init_run
|
97
|
+
@mq = Queue.new
|
98
|
+
@queue = Queue.new
|
99
|
+
@threads = {}
|
100
|
+
end
|
101
|
+
|
102
|
+
def queue_hosts(range)
|
103
|
+
range.each { |i| @queue << hosts[i] }
|
104
|
+
end
|
105
|
+
|
106
|
+
def spawn_queued_job
|
107
|
+
return if @queue.empty?
|
108
|
+
host = @queue.pop(false)
|
109
|
+
start_task_thread host
|
110
|
+
rescue ThreadError
|
111
|
+
end
|
112
|
+
|
113
|
+
def host_id_max_width
|
114
|
+
@host_id_max_width ||= hosts.map {|h| h[:id].size }.max
|
115
|
+
end
|
116
|
+
|
117
|
+
def report
|
118
|
+
@report ||= Report.new(hosts, exit_statuses, messages)
|
119
|
+
end
|
120
|
+
|
121
|
+
def wait_for_threads
|
122
|
+
@threads.each do |host, t|
|
123
|
+
@exit_statuses[host] = t.value
|
124
|
+
@messages[host] = t[:messages]
|
125
|
+
end
|
126
|
+
end
|
127
|
+
|
128
|
+
end
|
129
|
+
end
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: dply
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.1.
|
4
|
+
version: 0.1.17
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Neeraj
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2015-
|
11
|
+
date: 2015-05-03 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: ruby-elf
|
@@ -98,9 +98,11 @@ files:
|
|
98
98
|
- lib/dply/cli/build.rb
|
99
99
|
- lib/dply/cli/depcheck.rb
|
100
100
|
- lib/dply/cli/deploy.rb
|
101
|
+
- lib/dply/cli/devbuild.rb
|
101
102
|
- lib/dply/cli/install_pkgs.rb
|
102
103
|
- lib/dply/cli/reload.rb
|
103
104
|
- lib/dply/cli/reopen_logs.rb
|
105
|
+
- lib/dply/cli/setup.rb
|
104
106
|
- lib/dply/cli/status.rb
|
105
107
|
- lib/dply/cli/task.rb
|
106
108
|
- lib/dply/config.rb
|
@@ -128,6 +130,9 @@ files:
|
|
128
130
|
- lib/dply/strategy/archive.rb
|
129
131
|
- lib/dply/strategy/git.rb
|
130
132
|
- lib/dply/tasks.rb
|
133
|
+
- lib/dply/templates/build.erb
|
134
|
+
- lib/dply/templates/deploy.erb
|
135
|
+
- lib/dply/templates/pkgs.erb
|
131
136
|
- lib/dply/version.rb
|
132
137
|
- lib/dply/yum.rb
|
133
138
|
- lib/dplyr/cli.rb
|
@@ -135,6 +140,7 @@ files:
|
|
135
140
|
- lib/dplyr/report.rb
|
136
141
|
- lib/dplyr/stage.rb
|
137
142
|
- lib/dplyr/stages_config.rb
|
143
|
+
- lib/dplyr/task_runner.rb
|
138
144
|
- lib/dplyr/tasks_config.rb
|
139
145
|
homepage: ''
|
140
146
|
licenses:
|
@@ -156,7 +162,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
156
162
|
version: '0'
|
157
163
|
requirements: []
|
158
164
|
rubyforge_project:
|
159
|
-
rubygems_version: 2.
|
165
|
+
rubygems_version: 2.4.5
|
160
166
|
signing_key:
|
161
167
|
specification_version: 4
|
162
168
|
summary: rake based deploy tool
|