dply 0.1.13 → 0.1.17

Sign up to get free protection for your applications and to get access to all the features.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: 30096bb62002c6c0c805abbfbb25f21ee4257795
4
- data.tar.gz: ac5a69f5bdd143e77e04e1755cd1098fe35ce2c7
3
+ metadata.gz: e36e026a2a88ee74042f732298e42fb7f13a4eca
4
+ data.tar.gz: b82102df075f09482830a6e48850710a6788b09a
5
5
  SHA512:
6
- metadata.gz: 43a2fd54e8fea4a0ab99c48b9941b4a800bd23e9e4682880201544aadae1fd5fac51ccf66192f112d8faaf7142aeb8ab3ab82d87efca0f3e4fde2bc88abfa73e
7
- data.tar.gz: 3768616031dd0f8516bd87fdd423a6239b8ce22a6d94be2e206ca766444ac18284bbec3011b2fff6e1772462a68a9f8e76c013a1621ada62376debf68fd7d185
6
+ metadata.gz: e0d0e00e407aab69f5394aad3dc2e8833c41db1c2869eff653f5dd396cb681f1fd017e8211bf7795c9f1ce9001fba5b5f8885b113844bf7eae9d66b89ec999e8
7
+ data.tar.gz: 27983c97231765d2b2387c02e1f234fd8b060fcd2652389047be8cbeeb05cbd1e19e98380b8de502d2db0161740067a2dc3ef691f6733c2972dcf0367f495991
data/bin/dplyr CHANGED
@@ -45,10 +45,14 @@ opts_parser = OptionParser.new do |opts|
45
45
  logger.level = ::Logger::DEBUG
46
46
  end
47
47
 
48
+ opts.on("-m", "--enable-markers", "Enable marker logging") do
49
+ logger.enable_markers = true
50
+ end
51
+
48
52
  opts.on_tail("-h", "--help", "Show this message") do
49
53
  puts opts
50
54
  exit
51
- end
55
+ end
52
56
  end
53
57
  begin
54
58
  opts_parser.order!(ARGV)
@@ -70,4 +74,8 @@ rescue OptionParser::InvalidOption, OptionParser::MissingArgument, ::Dply::Error
70
74
  logger.error "#{e.message}"
71
75
  abort
72
76
  end
77
+ ensure
78
+ logger.marker "end"
73
79
  end
80
+
81
+
data/bin/drake CHANGED
@@ -71,7 +71,9 @@ begin
71
71
 
72
72
  command = (ARGV.shift || "").to_sym
73
73
  case command
74
- when :deploy, :reload, :task, :build, :depcheck, :"install-pkgs", :reopen_logs, :status
74
+ when :deploy, :reload, :task,
75
+ :build, :depcheck, :"install-pkgs", :reopen_logs,
76
+ :status, :devbuild, :setup
75
77
  run_cli command, ARGV
76
78
  when /\A[a-zA-Z_\-0-9]+[:][a-zA-Z_\-0-9]+\z/
77
79
  require 'dply/tasks'
data/docs/stages.rb CHANGED
@@ -1,8 +1,13 @@
1
+ # Sample stages.rb
2
+ # __________________________________________________________
3
+ #
4
+
1
5
  stage :production do
2
6
  user "dev"
3
- ask :user # exported to env as DPLY_USER
4
- deploy_dir "/home/dev/project"
7
+ dir "/home/dev/project"
5
8
  parallel_runs 1
6
- host "10.1.1.1", deploy_dir: "/home/dev/project1", id: "project1", user: "dev1"
7
- host "10.1.1.2", deploy_dir: "/home/dev/project2", id: "project2"
9
+ host "10.1.1.1", dir: "/home/dev/project1", id: "project1", user: "dev1" ## default roles: ["first"]
10
+ host "10.1.1.2", dir: "/home/dev/project2", id: "project2"
11
+ host "10.1.1.3", roles: ["db"]
12
+ host "10.1.1.4" ## default roles: ["last"]
8
13
  end
@@ -0,0 +1,67 @@
1
+ require 'dply/logger'
2
+ require 'dply/lock'
3
+ require 'dply/tasks'
4
+ require 'fileutils'
5
+
6
+ module Dply
7
+ module Cli
8
+ class Devbuild
9
+
10
+ include Logger
11
+
12
+ def initialize(argv)
13
+ @argv = argv
14
+ @options = {}
15
+ end
16
+
17
+ def run
18
+ lock.acquire
19
+ opts.parse!(@argv)
20
+ revision = @options[:revision] || "dev"
21
+ ENV["BUILD_NUMBER"] = revision
22
+
23
+ build_artifacts = "tmp/build_artifacts"
24
+ FileUtils.mkdir_p build_artifacts
25
+
26
+ clear_bundle_config
27
+ tasks.install_pkgs(build_mode: true, use_yum: @options[:use_yum])
28
+ tasks.build "build:default"
29
+ ensure
30
+ clear_bundle_config
31
+ end
32
+
33
+ def tasks
34
+ @tasks ||= ::Dply::Tasks.new
35
+ end
36
+
37
+ def lock
38
+ @lock ||= ::Dply::Lock.new
39
+ end
40
+
41
+ def clear_bundle_config
42
+ FileUtils.rm ".bundle/config" if File.exists? ".bundle/config"
43
+ end
44
+
45
+ def opts
46
+ OptionParser.new do |opts|
47
+
48
+ opts.banner = "Usage: drake devbuild [options] [target]"
49
+
50
+ opts.on("-r", "--revision [REVISION]", "Specify revision") do |r|
51
+ @options[:revision] = r
52
+ end
53
+
54
+ opts.on("--use-yum", "use yum to install packages") do |e|
55
+ @options[:use_yum] = true
56
+ end
57
+
58
+ opts.on("-h", "--help", "Help") do
59
+ puts opts
60
+ exit
61
+ end
62
+ end
63
+ end
64
+
65
+ end
66
+ end
67
+ end
@@ -0,0 +1,102 @@
1
+ require 'dply/logger'
2
+ require 'dply/lock'
3
+ require 'fileutils'
4
+ require 'erb'
5
+
6
+ module Dply
7
+ module Cli
8
+ class Setup
9
+
10
+ include ::Dply::Logger
11
+
12
+ def initialize(argv)
13
+ @argv = argv
14
+ @options = {}
15
+ end
16
+
17
+ def run
18
+ opts.parse!(@argv)
19
+ return if not proceed?
20
+ lock.acquire
21
+ setup_default
22
+ namespace = @argv.shift
23
+ if namespace
24
+ setup_task(namespace)
25
+ else
26
+ setup_build_task
27
+ setup_task(:production)
28
+ end
29
+ end
30
+
31
+ def proceed?
32
+ print "Are you sure?(y/n) "
33
+ v = STDIN.gets.strip
34
+ v == "y"
35
+ end
36
+
37
+ def setup_default
38
+ if not File.exist? "dply"
39
+ FileUtils.mkdir_p "dply"
40
+ logger.info "created dply/ dir"
41
+ else
42
+ logger.info "skipping dply/ dir"
43
+ end
44
+
45
+ rakefile = "dply/Rakefile"
46
+ if not File.exist? rakefile
47
+ FileUtils.touch rakefile
48
+ logger.info "created #{rakefile}"
49
+ else
50
+ logger.info "skipping #{rakefile}"
51
+ end
52
+
53
+ pkgs_yml = "pkgs.yml"
54
+ if not File.exist? pkgs_yml
55
+ FileUtils.cp "#{templates_dir}/pkgs.erb", pkgs_yml
56
+ logger.info "created #{pkgs_yml}"
57
+ else
58
+ logger.info "skipping #{pkgs_yml}"
59
+ end
60
+ end
61
+
62
+ def setup_task(namespace, template: "deploy.erb")
63
+ tasks_file = "dply/#{namespace}.rake"
64
+ if File.exist? tasks_file
65
+ logger.info "skipping #{tasks_file}"
66
+ return
67
+ end
68
+ b = binding
69
+ template_path = "#{templates_dir}/#{template}"
70
+ erb = ERB.new(File.read(template_path), nil, '-')
71
+ File.open(tasks_file, 'w') { |f| f.write erb.result(b) }
72
+ logger.info "created #{tasks_file}"
73
+ end
74
+
75
+ def setup_build_task
76
+ setup_task("build", template: "build.erb")
77
+ end
78
+
79
+ def templates_dir
80
+ @templates_dir ||= "#{__dir__}/../templates"
81
+ end
82
+
83
+ def lock
84
+ @lock ||= ::Dply::Lock.new
85
+ end
86
+
87
+ def opts
88
+ OptionParser.new do |opts|
89
+
90
+ opts.banner = "Usage: drake setup [options] [namespace]"
91
+
92
+ opts.on("-h", "--help", "Help") do
93
+ puts opts
94
+ exit
95
+ end
96
+ end
97
+ end
98
+
99
+
100
+ end
101
+ end
102
+ end
data/lib/dply/config.rb CHANGED
@@ -33,6 +33,10 @@ module Dply
33
33
  set :repo, repo
34
34
  end
35
35
 
36
+ def mirror(repo)
37
+ set :mirror, repo
38
+ end
39
+
36
40
  def branch(branch)
37
41
  set :branch, branch
38
42
  end
@@ -4,7 +4,7 @@ module Dply
4
4
 
5
5
  attr_writer :revision, :build_url, :build_url_proc,
6
6
  :revision_proc
7
- attr_accessor :dir, :name, :repo, :branch,
7
+ attr_accessor :dir, :name, :repo, :branch, :mirror,
8
8
  :strategy, :target, :verify_checksum,
9
9
  :config_map, :dir_map, :shared_dirs,
10
10
  :config_download_url, :config_skip_download
@@ -3,7 +3,7 @@ require 'dply/ext/string'
3
3
  module Dply
4
4
  class CustomLogger < ::Logger
5
5
 
6
- attr_writer :trace_mode, :remote_mode
6
+ attr_writer :trace_mode, :remote_mode, :enable_markers
7
7
 
8
8
  def initialize(file)
9
9
  super(file)
@@ -37,5 +37,10 @@ module Dply
37
37
  puts %{dply_msg|#{msg}}
38
38
  end
39
39
 
40
+ def marker(msg)
41
+ return if not @enable_markers
42
+ puts "dply_marker:#{msg}"
43
+ end
44
+
40
45
  end
41
46
  end
data/lib/dply/git.rb CHANGED
@@ -20,7 +20,12 @@ module Dply
20
20
  end
21
21
 
22
22
  def self.clone(repo, dir, mirror: nil)
23
- cmd "git clone #{repo} #{dir}"
23
+ if mirror
24
+ cmd "git clone #{mirror} #{dir}"
25
+ Dir.chdir(dir) { cmd "git remote set-url origin #{repo}" }
26
+ else
27
+ cmd "git clone #{repo} #{dir}"
28
+ end
24
29
  end
25
30
 
26
31
  def self.clean
@@ -29,7 +29,7 @@ module Dply
29
29
  logger.debug "skipping yum pkgs"
30
30
  return {}
31
31
  end
32
- YAML.safe_load(File.read(@pkgs_yml))
32
+ YAML.safe_load(File.read(@pkgs_yml)) || {}
33
33
  rescue => e
34
34
  error "error loading pkgs list"
35
35
  end
data/lib/dply/repo.rb CHANGED
@@ -14,7 +14,7 @@ module Dply
14
14
  if Dir.exist? "#{dir}/.git"
15
15
  raise "unable to create repo" if not verify_remote_url
16
16
  else
17
- Git.clone upstream, dir
17
+ Git.clone upstream, dir, mirror: @mirror
18
18
  end
19
19
  end
20
20
 
data/lib/dply/setup.rb CHANGED
@@ -36,7 +36,7 @@ module Dply
36
36
  private
37
37
 
38
38
  def create_repo
39
- repo = ::Dply::Repo.new("repo", @config.repo)
39
+ repo = ::Dply::Repo.new("repo", @config.repo, mirror: @config.mirror)
40
40
  repo.create
41
41
  end
42
42
 
data/lib/dply/tasks.rb CHANGED
@@ -4,6 +4,8 @@ require 'dply/bundle'
4
4
  require 'dply/yum'
5
5
  require 'dply/linker'
6
6
  require 'dply/pkgs_config'
7
+ require 'dply/error'
8
+ require 'etc'
7
9
 
8
10
  module Dply
9
11
  class Tasks
@@ -49,9 +51,12 @@ module Dply
49
51
  def install_pkgs(build_mode: false, use_yum: false)
50
52
  return if not File.exists? "pkgs.yml"
51
53
  drake_exists = File.exists? (drake_command)
54
+
55
+ pkgs = get_pkgs(build_mode)
52
56
  if use_yum || !drake_exists
53
- yum_install build_mode
57
+ yum_install pkgs
54
58
  else
59
+ return if Yum.new(pkgs).installed?
55
60
  command_install build_mode
56
61
  end
57
62
  end
@@ -70,20 +75,39 @@ module Dply
70
75
  @bundle ||= Bundle.new(deployment: @deployment)
71
76
  end
72
77
 
73
- def yum_install(build_mode)
74
- pkgs = PkgsConfig.new(build_mode: build_mode).pkgs
78
+ def get_pkgs(build_mode)
79
+ PkgsConfig.new(build_mode: build_mode).pkgs
80
+ end
81
+
82
+ def yum_install(pkgs)
75
83
  Yum.new(pkgs, sudo: true).install
76
84
  end
77
85
 
78
86
  def command_install(build_mode)
79
- command = "sudo -n #{drake_command} install-pkgs"
87
+ command = "#{drake_command} install-pkgs"
80
88
  command << " -b" if build_mode
81
- cmd command
89
+ check_sudo_permission command
90
+ cmd "sudo -n #{command}"
82
91
  end
83
92
 
84
93
  def drake_command
85
94
  @drake_command ||= (ENV["DRAKE_COMMAND"] || "/opt/ruby/bin/drake")
86
95
  end
87
96
 
97
+ def check_sudo_permission(command)
98
+ output = `sudo -n -l #{command}`
99
+ if output.chomp.strip == command
100
+ return true
101
+ else
102
+ msg = []
103
+ user = Etc.getpwuid(Process.uid).name
104
+ msg << %{unable to run "#{command}" with sudo permissions}
105
+ msg << %{To resolve add the following line to sudoers: }
106
+ msg << %{#{user} ALL=(ALL) NOPASSWD: /opt/ruby/bin/drake install-pkgs *, /opt/ruby/bin/drake install-pkgs}.yellow
107
+ raise Error, msg.join("\n")
108
+ end
109
+
110
+ end
111
+
88
112
  end
89
113
  end
@@ -0,0 +1,5 @@
1
+ namespace :build do
2
+ task :default do
3
+
4
+ end
5
+ end
@@ -0,0 +1,15 @@
1
+ namespace :<%= namespace %> do
2
+
3
+ task :deploy do
4
+
5
+ end
6
+
7
+ task :reload do
8
+
9
+ end
10
+
11
+ task :reopen_logs do
12
+
13
+ end
14
+
15
+ end
@@ -0,0 +1,5 @@
1
+ #pkgs:
2
+ # - postgresql94-libs
3
+ #
4
+ # build_pkgs:
5
+ # - postgresql94-devel
data/lib/dply/version.rb CHANGED
@@ -1,3 +1,3 @@
1
1
  module Dply
2
- VERSION = "0.1.13"
2
+ VERSION = "0.1.17"
3
3
  end
data/lib/dply/yum.rb CHANGED
@@ -26,6 +26,10 @@ module Dply
26
26
  end
27
27
  end
28
28
 
29
+ def installed?
30
+ not_installed_pkgs.size == 0
31
+ end
32
+
29
33
  private
30
34
 
31
35
  def pkgs_str
@@ -42,9 +46,5 @@ module Dply
42
46
  matches = `#{command}`.scan(/^package (.*) is not installed$/)
43
47
  end
44
48
 
45
- def installed?
46
- not_installed_pkgs.size == 0
47
- end
48
-
49
49
  end
50
50
  end
data/lib/dplyr/cli.rb CHANGED
@@ -1,5 +1,5 @@
1
1
  require 'dplyr/stages_config'
2
- require 'dplyr/remote_task'
2
+ require 'dplyr/task_runner'
3
3
  require 'dply/logger'
4
4
 
5
5
  module Dplyr
@@ -52,8 +52,8 @@ module Dplyr
52
52
  end
53
53
 
54
54
  def run_remote_task
55
- remote_task = RemoteTask.new(hosts, argv_str, parallel_jobs: parallel_jobs, env: env_str)
56
- remote_task.run
55
+ task_runner = TaskRunner.new(hosts, argv_str, parallel_jobs: parallel_jobs)
56
+ task_runner.run
57
57
  end
58
58
 
59
59
  end
@@ -6,85 +6,49 @@ module Dplyr
6
6
  class RemoteTask
7
7
 
8
8
  include ::Dply::Logger
9
-
10
- attr_reader :hosts, :parallel_jobs, :task, :env
11
9
 
12
- def initialize(hosts, task, parallel_jobs: 1, env: "")
13
- @hosts = hosts
14
- @parallel_jobs = parallel_jobs
10
+ attr_reader :exit_status, :messages
11
+
12
+ def initialize(host, task, id_size: nil)
13
+ @host = host
15
14
  @task = task
16
- @env = env
17
- @env << " PATH=/usr/sbin:/usr/local/sbin:$PATH"
15
+ @messages = []
16
+ @id_size = id_size || @host_info[:id].size
18
17
  end
19
18
 
20
19
  def run
21
- if parallel_jobs > 1 && hosts.count > 1
22
- run_in_parallel
23
- else
24
- run_serially
25
- end
20
+ reset!
21
+ r, w, pid = PTY.spawn(remote_cmd)
22
+ pty_read(r, @host[:id])
23
+ @exit_status = get_exit_status pid
26
24
  end
27
25
 
28
- def run_in_parallel
29
- init_run
30
- parallel_jobs.times do
31
- spawn_queued_job
32
- end
33
- loop do
34
- break if @queue.empty?
35
- @mq.pop
36
- spawn_queued_job
37
- end
38
- report.print_full
39
- end
40
-
41
- def run_serially
42
- hosts.each do |host_info|
43
- puts "=== Running on #{host_info[:id]} ==="
44
- run_cmd = remote_cmd host_info
45
- system run_cmd
46
- puts
47
- raise ::Dply::Error, "remote deploy failed on #{host_info[:id]}" if $? != 0
26
+ def remote_cmd
27
+ if logger.debug?
28
+ %(#{ssh} -l #{user} #{addr} '#{env} drake --remote --debug -d #{dir} #{@task} 2>&1')
29
+ else
30
+ %(#{ssh} -l #{user} #{addr} '#{env} drake --remote -d #{dir} #{@task} 2>&1' 2>/dev/null)
48
31
  end
49
32
  end
50
33
 
51
- private
52
-
53
- def init_run
54
- queue_all_hosts
55
- job_output_template
56
- @threads = {}
57
- @mq = Queue.new
34
+ def user
35
+ @host.fetch :user
58
36
  end
59
37
 
60
- def queue_all_hosts
61
- @queue = Queue.new
62
- hosts.each { |h| @queue << h }
38
+ def dir
39
+ @host.fetch :dir
63
40
  end
64
41
 
65
- def spawn_job(host_info)
66
- run_cmd = remote_cmd host_info
67
- thread = popen(run_cmd, host_info)
68
- @threads[host_info] = thread
42
+ def addr
43
+ @host.fetch :addr
69
44
  end
70
45
 
71
- def spawn_queued_job
72
- return if @queue.empty?
73
- host = @queue.pop(false)
74
- spawn_job host
75
- rescue ThreadError
46
+ def roles
47
+ @host.fetch :roles
76
48
  end
77
-
78
- def remote_cmd(host_info)
79
- user = host_info[:user]
80
- host = host_info[:host]
81
- dir = host_info[:dir]
82
- ssh = "ssh -tt -o BatchMode=yes -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null"
83
- if logger.debug?
84
- %(#{ssh} -l #{user} #{host} '#{env} drake --remote --debug -d #{dir} #{task} 2>&1')
85
- else
86
- %(#{ssh} -l #{user} #{host} '#{env} drake --remote -d #{dir} #{task} 2>&1' 2>/dev/null)
87
- end
49
+
50
+ def ssh
51
+ "ssh -tt -o BatchMode=yes -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null"
88
52
  end
89
53
 
90
54
  def pty_read(file, id)
@@ -92,72 +56,42 @@ module Dplyr
92
56
  if line =~ /\Adply_msg\|/
93
57
  receive_message line
94
58
  else
95
- printf @job_output_template, id, line
59
+ printf output_template, id, line
96
60
  end
97
61
  end
98
62
  rescue EOFError,Errno::ECONNRESET, Errno::EPIPE, Errno::EIO => e
99
63
  end
100
64
 
101
- def popen(cmd, host_info)
102
- t = Thread.new do |t|
103
- Thread.current[:messages] = []
104
- begin
105
- r, w, pid = PTY.spawn(cmd)
106
- pty_read(r, host_info[:id])
107
- exit_status pid
108
- ensure
109
- @mq << true
110
- end
111
- end
112
- t.abort_on_exception = true
113
- t.run
114
- return t
115
- end
116
-
117
- def host_id_max_width
118
- hosts.map {|h| h[:id].size }.max
119
- end
120
-
121
- def job_output_template
122
- @job_output_template ||= begin
123
- id_template = "%-#{host_id_max_width}s".bold.grey
124
- template = "#{id_template} %s"
125
- end
126
- end
127
-
128
65
  def receive_message(msg_str)
129
66
  msg = msg_str.partition("|")[2].strip
130
67
  return if msg.empty?
131
- messages = Thread.current[:messages]
132
- messages << msg
68
+ @messages << msg
133
69
  end
134
70
 
135
- def report
136
- @report ||= Report.new(hosts, exit_statuses, messages)
71
+ def get_exit_status(pid)
72
+ pid, status = Process.waitpid2(pid)
73
+ return status
137
74
  end
138
75
 
139
- def messages
140
- @messages ||= begin
141
- m = {}
142
- @threads.each do |host, thread|
143
- m[host] = thread[:messages]
144
- end
145
- m
76
+ def output_template
77
+ @output_template ||= begin
78
+ id_template = "%-#{@id_size}s".bold.grey
79
+ template = "#{id_template} %s"
146
80
  end
147
81
  end
148
82
 
149
- def exit_statuses
150
- return @exit_statuses if @exit_statuses
151
- @exit_statuses = {}
152
- @threads.each do |host, thread|
153
- @exit_statuses[host] = thread.value
154
- end
155
- @exit_statuses
83
+ def reset!
84
+ @output_template = nil
156
85
  end
157
86
 
158
- def exit_status(pid)
159
- pid, status = Process.waitpid2(pid)
160
- return status
87
+ def env
88
+ @env ||= "PATH=/usr/sbin:/usr/local/sbin:$PATH #{roles_env}"
89
+ end
90
+
91
+ def roles_env
92
+ return "" if not roles.size > 0
93
+ roles_str = roles.join(",")
94
+ "DPLY_ROLES=#{roles_str}"
161
95
  end
162
96
 
163
97
  end
data/lib/dplyr/report.rb CHANGED
@@ -1,6 +1,8 @@
1
+ require 'dply/logger'
1
2
  module Dplyr
2
3
  class Report
3
4
 
5
+ include ::Dply::Logger
4
6
  attr_reader :hosts, :exit_statuses
5
7
 
6
8
  def initialize(hosts, exit_statuses, messages)
@@ -11,6 +13,7 @@ module Dplyr
11
13
 
12
14
 
13
15
  def print_full
16
+ logger.marker "summary_start"
14
17
  print_successful_jobs
15
18
  print_failed_jobs
16
19
  print_summary
@@ -52,7 +55,7 @@ module Dplyr
52
55
  run_count = @exit_statuses.count
53
56
  not_run = total_hosts - run_count
54
57
  if (failed.count > 0 || not_run > 0 )
55
- not_run_error = "not run on #{not_run}/#{total_hosts}" if not_run > 0
58
+ not_run_error = "not run on #{not_run} of #{total_hosts} hosts" if not_run > 0
56
59
  failed_error = "failed on #{failed.count} of #{total_hosts} hosts" if failed.count > 0
57
60
 
58
61
  errors = []
data/lib/dplyr/stage.rb CHANGED
@@ -1,6 +1,10 @@
1
+ require 'dply/helper'
2
+
1
3
  module Dplyr
2
4
  class Stage
3
5
 
6
+ include ::Dply::Helper
7
+
4
8
  attr_accessor :config_proc
5
9
 
6
10
  def initialize(name)
@@ -18,12 +22,13 @@ module Dplyr
18
22
  })
19
23
  end
20
24
 
21
- def host(host, user: nil, dir: nil, id: nil)
25
+ def host(addr, user: nil, dir: nil, id: nil, roles: [])
22
26
  @hosts << ({
23
- host: host,
27
+ addr: addr,
24
28
  user: user,
25
29
  dir: dir,
26
- id: id || host
30
+ id: id || addr,
31
+ roles: cleaned_roles(roles)
27
32
  })
28
33
  end
29
34
 
@@ -54,15 +59,31 @@ module Dplyr
54
59
  return value
55
60
  end
56
61
 
62
+ def add_default_roles
63
+ return if not @hosts.size > 0
64
+ @hosts.first[:roles] << "first"
65
+ @hosts.last[:roles] << "last"
66
+ end
67
+
57
68
  def finalize
58
69
  return if @finalized
59
70
  instance_eval &config_proc if config_proc
60
71
  fill_hosts
72
+ add_default_roles
61
73
  @hosts.freeze
62
74
  @parallel_runs.freeze
63
75
  @finalized = true
64
76
  end
65
77
 
78
+ def cleaned_roles(roles)
79
+ error "roles must be an array" if not roles.is_a? Array
80
+ roles.each do |r|
81
+ r.strip!
82
+ error "invalid role value #{r} " if not r =~ /\A[0-9A-Za-z_\-]+\z/
83
+ end
84
+ return roles
85
+ end
86
+
66
87
  end
67
88
  end
68
89
 
@@ -0,0 +1,129 @@
1
+ require 'dplyr/report'
2
+ require 'dply/logger'
3
+ require 'dplyr/remote_task'
4
+
5
+ module Dplyr
6
+ class TaskRunner
7
+
8
+ include ::Dply::Logger
9
+
10
+ attr_reader :hosts, :parallel_jobs, :task, :messages, :exit_statuses
11
+ attr_writer :auto_serialize
12
+
13
+ def initialize(hosts, task, parallel_jobs: 1)
14
+ @hosts = hosts
15
+ @parallel_jobs = parallel_jobs
16
+ @task = task
17
+
18
+ @messages = {}
19
+ @exit_statuses = {}
20
+ @auto_serialize = true
21
+ end
22
+
23
+ def run
24
+ if parallel_jobs > 1 && hosts.count > 1
25
+ run_in_parallel
26
+ else
27
+ run_serially
28
+ end
29
+ report.print_full
30
+ end
31
+
32
+ def run_serially
33
+ hosts.each do |host|
34
+ task = execute_serially host
35
+ break if task.exit_status != 0
36
+ end
37
+ end
38
+
39
+ def run_in_parallel
40
+ if @auto_serialize
41
+ execute_serially hosts[0]
42
+ execute_in_parallel Range.new(1,hosts.size - 2)
43
+ execute_serially hosts[-1]
44
+ else
45
+ execute_in_parallel Range.new(0, hosts.size - 1)
46
+ end
47
+ end
48
+
49
+ private
50
+
51
+ def remote_task(host)
52
+ RemoteTask.new(host, @task, id_size: host_id_max_width)
53
+ end
54
+
55
+ def execute_serially(host)
56
+ task = remote_task host
57
+ task.run
58
+ @exit_statuses[host] = task.exit_status
59
+ @messages[host] = task.messages
60
+ return task
61
+ end
62
+
63
+ def execute_in_parallel(range)
64
+ init_run
65
+ queue_hosts range
66
+ parallel_jobs.times do
67
+ spawn_queued_job
68
+ end
69
+ loop do
70
+ break if @queue.empty?
71
+ @mq.pop
72
+ spawn_queued_job
73
+ end
74
+ wait_for_threads
75
+ end
76
+
77
+ def start_task_thread(host)
78
+ t = Thread.new do
79
+ Thread.current[:messages] = []
80
+ task = remote_task(host)
81
+ begin
82
+ task.run
83
+ Thread.current[:messages] = task.messages
84
+ Thread.current[:exit_status] = task.exit_status
85
+ rescue => e
86
+ puts e.message
87
+ ensure
88
+ @mq << true
89
+ end
90
+ end
91
+ t.abort_on_exception = true
92
+ t.run
93
+ @threads[host] = t
94
+ end
95
+
96
+ def init_run
97
+ @mq = Queue.new
98
+ @queue = Queue.new
99
+ @threads = {}
100
+ end
101
+
102
+ def queue_hosts(range)
103
+ range.each { |i| @queue << hosts[i] }
104
+ end
105
+
106
+ def spawn_queued_job
107
+ return if @queue.empty?
108
+ host = @queue.pop(false)
109
+ start_task_thread host
110
+ rescue ThreadError
111
+ end
112
+
113
+ def host_id_max_width
114
+ @host_id_max_width ||= hosts.map {|h| h[:id].size }.max
115
+ end
116
+
117
+ def report
118
+ @report ||= Report.new(hosts, exit_statuses, messages)
119
+ end
120
+
121
+ def wait_for_threads
122
+ @threads.each do |host, t|
123
+ @exit_statuses[host] = t.value
124
+ @messages[host] = t[:messages]
125
+ end
126
+ end
127
+
128
+ end
129
+ end
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: dply
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.1.13
4
+ version: 0.1.17
5
5
  platform: ruby
6
6
  authors:
7
7
  - Neeraj
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2015-04-13 00:00:00.000000000 Z
11
+ date: 2015-05-03 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: ruby-elf
@@ -98,9 +98,11 @@ files:
98
98
  - lib/dply/cli/build.rb
99
99
  - lib/dply/cli/depcheck.rb
100
100
  - lib/dply/cli/deploy.rb
101
+ - lib/dply/cli/devbuild.rb
101
102
  - lib/dply/cli/install_pkgs.rb
102
103
  - lib/dply/cli/reload.rb
103
104
  - lib/dply/cli/reopen_logs.rb
105
+ - lib/dply/cli/setup.rb
104
106
  - lib/dply/cli/status.rb
105
107
  - lib/dply/cli/task.rb
106
108
  - lib/dply/config.rb
@@ -128,6 +130,9 @@ files:
128
130
  - lib/dply/strategy/archive.rb
129
131
  - lib/dply/strategy/git.rb
130
132
  - lib/dply/tasks.rb
133
+ - lib/dply/templates/build.erb
134
+ - lib/dply/templates/deploy.erb
135
+ - lib/dply/templates/pkgs.erb
131
136
  - lib/dply/version.rb
132
137
  - lib/dply/yum.rb
133
138
  - lib/dplyr/cli.rb
@@ -135,6 +140,7 @@ files:
135
140
  - lib/dplyr/report.rb
136
141
  - lib/dplyr/stage.rb
137
142
  - lib/dplyr/stages_config.rb
143
+ - lib/dplyr/task_runner.rb
138
144
  - lib/dplyr/tasks_config.rb
139
145
  homepage: ''
140
146
  licenses:
@@ -156,7 +162,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
156
162
  version: '0'
157
163
  requirements: []
158
164
  rubyforge_project:
159
- rubygems_version: 2.2.2
165
+ rubygems_version: 2.4.5
160
166
  signing_key:
161
167
  specification_version: 4
162
168
  summary: rake based deploy tool