dply 0.1.13 → 0.1.17

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: 30096bb62002c6c0c805abbfbb25f21ee4257795
4
- data.tar.gz: ac5a69f5bdd143e77e04e1755cd1098fe35ce2c7
3
+ metadata.gz: e36e026a2a88ee74042f732298e42fb7f13a4eca
4
+ data.tar.gz: b82102df075f09482830a6e48850710a6788b09a
5
5
  SHA512:
6
- metadata.gz: 43a2fd54e8fea4a0ab99c48b9941b4a800bd23e9e4682880201544aadae1fd5fac51ccf66192f112d8faaf7142aeb8ab3ab82d87efca0f3e4fde2bc88abfa73e
7
- data.tar.gz: 3768616031dd0f8516bd87fdd423a6239b8ce22a6d94be2e206ca766444ac18284bbec3011b2fff6e1772462a68a9f8e76c013a1621ada62376debf68fd7d185
6
+ metadata.gz: e0d0e00e407aab69f5394aad3dc2e8833c41db1c2869eff653f5dd396cb681f1fd017e8211bf7795c9f1ce9001fba5b5f8885b113844bf7eae9d66b89ec999e8
7
+ data.tar.gz: 27983c97231765d2b2387c02e1f234fd8b060fcd2652389047be8cbeeb05cbd1e19e98380b8de502d2db0161740067a2dc3ef691f6733c2972dcf0367f495991
data/bin/dplyr CHANGED
@@ -45,10 +45,14 @@ opts_parser = OptionParser.new do |opts|
45
45
  logger.level = ::Logger::DEBUG
46
46
  end
47
47
 
48
+ opts.on("-m", "--enable-markers", "Enable marker logging") do
49
+ logger.enable_markers = true
50
+ end
51
+
48
52
  opts.on_tail("-h", "--help", "Show this message") do
49
53
  puts opts
50
54
  exit
51
- end
55
+ end
52
56
  end
53
57
  begin
54
58
  opts_parser.order!(ARGV)
@@ -70,4 +74,8 @@ rescue OptionParser::InvalidOption, OptionParser::MissingArgument, ::Dply::Error
70
74
  logger.error "#{e.message}"
71
75
  abort
72
76
  end
77
+ ensure
78
+ logger.marker "end"
73
79
  end
80
+
81
+
data/bin/drake CHANGED
@@ -71,7 +71,9 @@ begin
71
71
 
72
72
  command = (ARGV.shift || "").to_sym
73
73
  case command
74
- when :deploy, :reload, :task, :build, :depcheck, :"install-pkgs", :reopen_logs, :status
74
+ when :deploy, :reload, :task,
75
+ :build, :depcheck, :"install-pkgs", :reopen_logs,
76
+ :status, :devbuild, :setup
75
77
  run_cli command, ARGV
76
78
  when /\A[a-zA-Z_\-0-9]+[:][a-zA-Z_\-0-9]+\z/
77
79
  require 'dply/tasks'
data/docs/stages.rb CHANGED
@@ -1,8 +1,13 @@
1
+ # Sample stages.rb
2
+ # __________________________________________________________
3
+ #
4
+
1
5
  stage :production do
2
6
  user "dev"
3
- ask :user # exported to env as DPLY_USER
4
- deploy_dir "/home/dev/project"
7
+ dir "/home/dev/project"
5
8
  parallel_runs 1
6
- host "10.1.1.1", deploy_dir: "/home/dev/project1", id: "project1", user: "dev1"
7
- host "10.1.1.2", deploy_dir: "/home/dev/project2", id: "project2"
9
+ host "10.1.1.1", dir: "/home/dev/project1", id: "project1", user: "dev1" ## default roles: ["first"]
10
+ host "10.1.1.2", dir: "/home/dev/project2", id: "project2"
11
+ host "10.1.1.3", roles: ["db"]
12
+ host "10.1.1.4" ## default roles: ["last"]
8
13
  end
@@ -0,0 +1,67 @@
1
+ require 'dply/logger'
2
+ require 'dply/lock'
3
+ require 'dply/tasks'
4
+ require 'fileutils'
5
+
6
+ module Dply
7
+ module Cli
8
+ class Devbuild
9
+
10
+ include Logger
11
+
12
+ def initialize(argv)
13
+ @argv = argv
14
+ @options = {}
15
+ end
16
+
17
+ def run
18
+ lock.acquire
19
+ opts.parse!(@argv)
20
+ revision = @options[:revision] || "dev"
21
+ ENV["BUILD_NUMBER"] = revision
22
+
23
+ build_artifacts = "tmp/build_artifacts"
24
+ FileUtils.mkdir_p build_artifacts
25
+
26
+ clear_bundle_config
27
+ tasks.install_pkgs(build_mode: true, use_yum: @options[:use_yum])
28
+ tasks.build "build:default"
29
+ ensure
30
+ clear_bundle_config
31
+ end
32
+
33
+ def tasks
34
+ @tasks ||= ::Dply::Tasks.new
35
+ end
36
+
37
+ def lock
38
+ @lock ||= ::Dply::Lock.new
39
+ end
40
+
41
+ def clear_bundle_config
42
+ FileUtils.rm ".bundle/config" if File.exists? ".bundle/config"
43
+ end
44
+
45
+ def opts
46
+ OptionParser.new do |opts|
47
+
48
+ opts.banner = "Usage: drake devbuild [options] [target]"
49
+
50
+ opts.on("-r", "--revision [REVISION]", "Specify revision") do |r|
51
+ @options[:revision] = r
52
+ end
53
+
54
+ opts.on("--use-yum", "use yum to install packages") do |e|
55
+ @options[:use_yum] = true
56
+ end
57
+
58
+ opts.on("-h", "--help", "Help") do
59
+ puts opts
60
+ exit
61
+ end
62
+ end
63
+ end
64
+
65
+ end
66
+ end
67
+ end
@@ -0,0 +1,102 @@
1
+ require 'dply/logger'
2
+ require 'dply/lock'
3
+ require 'fileutils'
4
+ require 'erb'
5
+
6
+ module Dply
7
+ module Cli
8
+ class Setup
9
+
10
+ include ::Dply::Logger
11
+
12
+ def initialize(argv)
13
+ @argv = argv
14
+ @options = {}
15
+ end
16
+
17
+ def run
18
+ opts.parse!(@argv)
19
+ return if not proceed?
20
+ lock.acquire
21
+ setup_default
22
+ namespace = @argv.shift
23
+ if namespace
24
+ setup_task(namespace)
25
+ else
26
+ setup_build_task
27
+ setup_task(:production)
28
+ end
29
+ end
30
+
31
+ def proceed?
32
+ print "Are you sure?(y/n) "
33
+ v = STDIN.gets.strip
34
+ v == "y"
35
+ end
36
+
37
+ def setup_default
38
+ if not File.exist? "dply"
39
+ FileUtils.mkdir_p "dply"
40
+ logger.info "created dply/ dir"
41
+ else
42
+ logger.info "skipping dply/ dir"
43
+ end
44
+
45
+ rakefile = "dply/Rakefile"
46
+ if not File.exist? rakefile
47
+ FileUtils.touch rakefile
48
+ logger.info "created #{rakefile}"
49
+ else
50
+ logger.info "skipping #{rakefile}"
51
+ end
52
+
53
+ pkgs_yml = "pkgs.yml"
54
+ if not File.exist? pkgs_yml
55
+ FileUtils.cp "#{templates_dir}/pkgs.erb", pkgs_yml
56
+ logger.info "created #{pkgs_yml}"
57
+ else
58
+ logger.info "skipping #{pkgs_yml}"
59
+ end
60
+ end
61
+
62
+ def setup_task(namespace, template: "deploy.erb")
63
+ tasks_file = "dply/#{namespace}.rake"
64
+ if File.exist? tasks_file
65
+ logger.info "skipping #{tasks_file}"
66
+ return
67
+ end
68
+ b = binding
69
+ template_path = "#{templates_dir}/#{template}"
70
+ erb = ERB.new(File.read(template_path), nil, '-')
71
+ File.open(tasks_file, 'w') { |f| f.write erb.result(b) }
72
+ logger.info "created #{tasks_file}"
73
+ end
74
+
75
+ def setup_build_task
76
+ setup_task("build", template: "build.erb")
77
+ end
78
+
79
+ def templates_dir
80
+ @templates_dir ||= "#{__dir__}/../templates"
81
+ end
82
+
83
+ def lock
84
+ @lock ||= ::Dply::Lock.new
85
+ end
86
+
87
+ def opts
88
+ OptionParser.new do |opts|
89
+
90
+ opts.banner = "Usage: drake setup [options] [namespace]"
91
+
92
+ opts.on("-h", "--help", "Help") do
93
+ puts opts
94
+ exit
95
+ end
96
+ end
97
+ end
98
+
99
+
100
+ end
101
+ end
102
+ end
data/lib/dply/config.rb CHANGED
@@ -33,6 +33,10 @@ module Dply
33
33
  set :repo, repo
34
34
  end
35
35
 
36
+ def mirror(repo)
37
+ set :mirror, repo
38
+ end
39
+
36
40
  def branch(branch)
37
41
  set :branch, branch
38
42
  end
@@ -4,7 +4,7 @@ module Dply
4
4
 
5
5
  attr_writer :revision, :build_url, :build_url_proc,
6
6
  :revision_proc
7
- attr_accessor :dir, :name, :repo, :branch,
7
+ attr_accessor :dir, :name, :repo, :branch, :mirror,
8
8
  :strategy, :target, :verify_checksum,
9
9
  :config_map, :dir_map, :shared_dirs,
10
10
  :config_download_url, :config_skip_download
@@ -3,7 +3,7 @@ require 'dply/ext/string'
3
3
  module Dply
4
4
  class CustomLogger < ::Logger
5
5
 
6
- attr_writer :trace_mode, :remote_mode
6
+ attr_writer :trace_mode, :remote_mode, :enable_markers
7
7
 
8
8
  def initialize(file)
9
9
  super(file)
@@ -37,5 +37,10 @@ module Dply
37
37
  puts %{dply_msg|#{msg}}
38
38
  end
39
39
 
40
+ def marker(msg)
41
+ return if not @enable_markers
42
+ puts "dply_marker:#{msg}"
43
+ end
44
+
40
45
  end
41
46
  end
data/lib/dply/git.rb CHANGED
@@ -20,7 +20,12 @@ module Dply
20
20
  end
21
21
 
22
22
  def self.clone(repo, dir, mirror: nil)
23
- cmd "git clone #{repo} #{dir}"
23
+ if mirror
24
+ cmd "git clone #{mirror} #{dir}"
25
+ Dir.chdir(dir) { cmd "git remote set-url origin #{repo}" }
26
+ else
27
+ cmd "git clone #{repo} #{dir}"
28
+ end
24
29
  end
25
30
 
26
31
  def self.clean
@@ -29,7 +29,7 @@ module Dply
29
29
  logger.debug "skipping yum pkgs"
30
30
  return {}
31
31
  end
32
- YAML.safe_load(File.read(@pkgs_yml))
32
+ YAML.safe_load(File.read(@pkgs_yml)) || {}
33
33
  rescue => e
34
34
  error "error loading pkgs list"
35
35
  end
data/lib/dply/repo.rb CHANGED
@@ -14,7 +14,7 @@ module Dply
14
14
  if Dir.exist? "#{dir}/.git"
15
15
  raise "unable to create repo" if not verify_remote_url
16
16
  else
17
- Git.clone upstream, dir
17
+ Git.clone upstream, dir, mirror: @mirror
18
18
  end
19
19
  end
20
20
 
data/lib/dply/setup.rb CHANGED
@@ -36,7 +36,7 @@ module Dply
36
36
  private
37
37
 
38
38
  def create_repo
39
- repo = ::Dply::Repo.new("repo", @config.repo)
39
+ repo = ::Dply::Repo.new("repo", @config.repo, mirror: @config.mirror)
40
40
  repo.create
41
41
  end
42
42
 
data/lib/dply/tasks.rb CHANGED
@@ -4,6 +4,8 @@ require 'dply/bundle'
4
4
  require 'dply/yum'
5
5
  require 'dply/linker'
6
6
  require 'dply/pkgs_config'
7
+ require 'dply/error'
8
+ require 'etc'
7
9
 
8
10
  module Dply
9
11
  class Tasks
@@ -49,9 +51,12 @@ module Dply
49
51
  def install_pkgs(build_mode: false, use_yum: false)
50
52
  return if not File.exists? "pkgs.yml"
51
53
  drake_exists = File.exists? (drake_command)
54
+
55
+ pkgs = get_pkgs(build_mode)
52
56
  if use_yum || !drake_exists
53
- yum_install build_mode
57
+ yum_install pkgs
54
58
  else
59
+ return if Yum.new(pkgs).installed?
55
60
  command_install build_mode
56
61
  end
57
62
  end
@@ -70,20 +75,39 @@ module Dply
70
75
  @bundle ||= Bundle.new(deployment: @deployment)
71
76
  end
72
77
 
73
- def yum_install(build_mode)
74
- pkgs = PkgsConfig.new(build_mode: build_mode).pkgs
78
+ def get_pkgs(build_mode)
79
+ PkgsConfig.new(build_mode: build_mode).pkgs
80
+ end
81
+
82
+ def yum_install(pkgs)
75
83
  Yum.new(pkgs, sudo: true).install
76
84
  end
77
85
 
78
86
  def command_install(build_mode)
79
- command = "sudo -n #{drake_command} install-pkgs"
87
+ command = "#{drake_command} install-pkgs"
80
88
  command << " -b" if build_mode
81
- cmd command
89
+ check_sudo_permission command
90
+ cmd "sudo -n #{command}"
82
91
  end
83
92
 
84
93
  def drake_command
85
94
  @drake_command ||= (ENV["DRAKE_COMMAND"] || "/opt/ruby/bin/drake")
86
95
  end
87
96
 
97
+ def check_sudo_permission(command)
98
+ output = `sudo -n -l #{command}`
99
+ if output.chomp.strip == command
100
+ return true
101
+ else
102
+ msg = []
103
+ user = Etc.getpwuid(Process.uid).name
104
+ msg << %{unable to run "#{command}" with sudo permissions}
105
+ msg << %{To resolve add the following line to sudoers: }
106
+ msg << %{#{user} ALL=(ALL) NOPASSWD: /opt/ruby/bin/drake install-pkgs *, /opt/ruby/bin/drake install-pkgs}.yellow
107
+ raise Error, msg.join("\n")
108
+ end
109
+
110
+ end
111
+
88
112
  end
89
113
  end
@@ -0,0 +1,5 @@
1
+ namespace :build do
2
+ task :default do
3
+
4
+ end
5
+ end
@@ -0,0 +1,15 @@
1
+ namespace :<%= namespace %> do
2
+
3
+ task :deploy do
4
+
5
+ end
6
+
7
+ task :reload do
8
+
9
+ end
10
+
11
+ task :reopen_logs do
12
+
13
+ end
14
+
15
+ end
@@ -0,0 +1,5 @@
1
+ #pkgs:
2
+ # - postgresql94-libs
3
+ #
4
+ # build_pkgs:
5
+ # - postgresql94-devel
data/lib/dply/version.rb CHANGED
@@ -1,3 +1,3 @@
1
1
  module Dply
2
- VERSION = "0.1.13"
2
+ VERSION = "0.1.17"
3
3
  end
data/lib/dply/yum.rb CHANGED
@@ -26,6 +26,10 @@ module Dply
26
26
  end
27
27
  end
28
28
 
29
+ def installed?
30
+ not_installed_pkgs.size == 0
31
+ end
32
+
29
33
  private
30
34
 
31
35
  def pkgs_str
@@ -42,9 +46,5 @@ module Dply
42
46
  matches = `#{command}`.scan(/^package (.*) is not installed$/)
43
47
  end
44
48
 
45
- def installed?
46
- not_installed_pkgs.size == 0
47
- end
48
-
49
49
  end
50
50
  end
data/lib/dplyr/cli.rb CHANGED
@@ -1,5 +1,5 @@
1
1
  require 'dplyr/stages_config'
2
- require 'dplyr/remote_task'
2
+ require 'dplyr/task_runner'
3
3
  require 'dply/logger'
4
4
 
5
5
  module Dplyr
@@ -52,8 +52,8 @@ module Dplyr
52
52
  end
53
53
 
54
54
  def run_remote_task
55
- remote_task = RemoteTask.new(hosts, argv_str, parallel_jobs: parallel_jobs, env: env_str)
56
- remote_task.run
55
+ task_runner = TaskRunner.new(hosts, argv_str, parallel_jobs: parallel_jobs)
56
+ task_runner.run
57
57
  end
58
58
 
59
59
  end
@@ -6,85 +6,49 @@ module Dplyr
6
6
  class RemoteTask
7
7
 
8
8
  include ::Dply::Logger
9
-
10
- attr_reader :hosts, :parallel_jobs, :task, :env
11
9
 
12
- def initialize(hosts, task, parallel_jobs: 1, env: "")
13
- @hosts = hosts
14
- @parallel_jobs = parallel_jobs
10
+ attr_reader :exit_status, :messages
11
+
12
+ def initialize(host, task, id_size: nil)
13
+ @host = host
15
14
  @task = task
16
- @env = env
17
- @env << " PATH=/usr/sbin:/usr/local/sbin:$PATH"
15
+ @messages = []
16
+ @id_size = id_size || @host_info[:id].size
18
17
  end
19
18
 
20
19
  def run
21
- if parallel_jobs > 1 && hosts.count > 1
22
- run_in_parallel
23
- else
24
- run_serially
25
- end
20
+ reset!
21
+ r, w, pid = PTY.spawn(remote_cmd)
22
+ pty_read(r, @host[:id])
23
+ @exit_status = get_exit_status pid
26
24
  end
27
25
 
28
- def run_in_parallel
29
- init_run
30
- parallel_jobs.times do
31
- spawn_queued_job
32
- end
33
- loop do
34
- break if @queue.empty?
35
- @mq.pop
36
- spawn_queued_job
37
- end
38
- report.print_full
39
- end
40
-
41
- def run_serially
42
- hosts.each do |host_info|
43
- puts "=== Running on #{host_info[:id]} ==="
44
- run_cmd = remote_cmd host_info
45
- system run_cmd
46
- puts
47
- raise ::Dply::Error, "remote deploy failed on #{host_info[:id]}" if $? != 0
26
+ def remote_cmd
27
+ if logger.debug?
28
+ %(#{ssh} -l #{user} #{addr} '#{env} drake --remote --debug -d #{dir} #{@task} 2>&1')
29
+ else
30
+ %(#{ssh} -l #{user} #{addr} '#{env} drake --remote -d #{dir} #{@task} 2>&1' 2>/dev/null)
48
31
  end
49
32
  end
50
33
 
51
- private
52
-
53
- def init_run
54
- queue_all_hosts
55
- job_output_template
56
- @threads = {}
57
- @mq = Queue.new
34
+ def user
35
+ @host.fetch :user
58
36
  end
59
37
 
60
- def queue_all_hosts
61
- @queue = Queue.new
62
- hosts.each { |h| @queue << h }
38
+ def dir
39
+ @host.fetch :dir
63
40
  end
64
41
 
65
- def spawn_job(host_info)
66
- run_cmd = remote_cmd host_info
67
- thread = popen(run_cmd, host_info)
68
- @threads[host_info] = thread
42
+ def addr
43
+ @host.fetch :addr
69
44
  end
70
45
 
71
- def spawn_queued_job
72
- return if @queue.empty?
73
- host = @queue.pop(false)
74
- spawn_job host
75
- rescue ThreadError
46
+ def roles
47
+ @host.fetch :roles
76
48
  end
77
-
78
- def remote_cmd(host_info)
79
- user = host_info[:user]
80
- host = host_info[:host]
81
- dir = host_info[:dir]
82
- ssh = "ssh -tt -o BatchMode=yes -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null"
83
- if logger.debug?
84
- %(#{ssh} -l #{user} #{host} '#{env} drake --remote --debug -d #{dir} #{task} 2>&1')
85
- else
86
- %(#{ssh} -l #{user} #{host} '#{env} drake --remote -d #{dir} #{task} 2>&1' 2>/dev/null)
87
- end
49
+
50
+ def ssh
51
+ "ssh -tt -o BatchMode=yes -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null"
88
52
  end
89
53
 
90
54
  def pty_read(file, id)
@@ -92,72 +56,42 @@ module Dplyr
92
56
  if line =~ /\Adply_msg\|/
93
57
  receive_message line
94
58
  else
95
- printf @job_output_template, id, line
59
+ printf output_template, id, line
96
60
  end
97
61
  end
98
62
  rescue EOFError,Errno::ECONNRESET, Errno::EPIPE, Errno::EIO => e
99
63
  end
100
64
 
101
- def popen(cmd, host_info)
102
- t = Thread.new do |t|
103
- Thread.current[:messages] = []
104
- begin
105
- r, w, pid = PTY.spawn(cmd)
106
- pty_read(r, host_info[:id])
107
- exit_status pid
108
- ensure
109
- @mq << true
110
- end
111
- end
112
- t.abort_on_exception = true
113
- t.run
114
- return t
115
- end
116
-
117
- def host_id_max_width
118
- hosts.map {|h| h[:id].size }.max
119
- end
120
-
121
- def job_output_template
122
- @job_output_template ||= begin
123
- id_template = "%-#{host_id_max_width}s".bold.grey
124
- template = "#{id_template} %s"
125
- end
126
- end
127
-
128
65
  def receive_message(msg_str)
129
66
  msg = msg_str.partition("|")[2].strip
130
67
  return if msg.empty?
131
- messages = Thread.current[:messages]
132
- messages << msg
68
+ @messages << msg
133
69
  end
134
70
 
135
- def report
136
- @report ||= Report.new(hosts, exit_statuses, messages)
71
+ def get_exit_status(pid)
72
+ pid, status = Process.waitpid2(pid)
73
+ return status
137
74
  end
138
75
 
139
- def messages
140
- @messages ||= begin
141
- m = {}
142
- @threads.each do |host, thread|
143
- m[host] = thread[:messages]
144
- end
145
- m
76
+ def output_template
77
+ @output_template ||= begin
78
+ id_template = "%-#{@id_size}s".bold.grey
79
+ template = "#{id_template} %s"
146
80
  end
147
81
  end
148
82
 
149
- def exit_statuses
150
- return @exit_statuses if @exit_statuses
151
- @exit_statuses = {}
152
- @threads.each do |host, thread|
153
- @exit_statuses[host] = thread.value
154
- end
155
- @exit_statuses
83
+ def reset!
84
+ @output_template = nil
156
85
  end
157
86
 
158
- def exit_status(pid)
159
- pid, status = Process.waitpid2(pid)
160
- return status
87
+ def env
88
+ @env ||= "PATH=/usr/sbin:/usr/local/sbin:$PATH #{roles_env}"
89
+ end
90
+
91
+ def roles_env
92
+ return "" if not roles.size > 0
93
+ roles_str = roles.join(",")
94
+ "DPLY_ROLES=#{roles_str}"
161
95
  end
162
96
 
163
97
  end
data/lib/dplyr/report.rb CHANGED
@@ -1,6 +1,8 @@
1
+ require 'dply/logger'
1
2
  module Dplyr
2
3
  class Report
3
4
 
5
+ include ::Dply::Logger
4
6
  attr_reader :hosts, :exit_statuses
5
7
 
6
8
  def initialize(hosts, exit_statuses, messages)
@@ -11,6 +13,7 @@ module Dplyr
11
13
 
12
14
 
13
15
  def print_full
16
+ logger.marker "summary_start"
14
17
  print_successful_jobs
15
18
  print_failed_jobs
16
19
  print_summary
@@ -52,7 +55,7 @@ module Dplyr
52
55
  run_count = @exit_statuses.count
53
56
  not_run = total_hosts - run_count
54
57
  if (failed.count > 0 || not_run > 0 )
55
- not_run_error = "not run on #{not_run}/#{total_hosts}" if not_run > 0
58
+ not_run_error = "not run on #{not_run} of #{total_hosts} hosts" if not_run > 0
56
59
  failed_error = "failed on #{failed.count} of #{total_hosts} hosts" if failed.count > 0
57
60
 
58
61
  errors = []
data/lib/dplyr/stage.rb CHANGED
@@ -1,6 +1,10 @@
1
+ require 'dply/helper'
2
+
1
3
  module Dplyr
2
4
  class Stage
3
5
 
6
+ include ::Dply::Helper
7
+
4
8
  attr_accessor :config_proc
5
9
 
6
10
  def initialize(name)
@@ -18,12 +22,13 @@ module Dplyr
18
22
  })
19
23
  end
20
24
 
21
- def host(host, user: nil, dir: nil, id: nil)
25
+ def host(addr, user: nil, dir: nil, id: nil, roles: [])
22
26
  @hosts << ({
23
- host: host,
27
+ addr: addr,
24
28
  user: user,
25
29
  dir: dir,
26
- id: id || host
30
+ id: id || addr,
31
+ roles: cleaned_roles(roles)
27
32
  })
28
33
  end
29
34
 
@@ -54,15 +59,31 @@ module Dplyr
54
59
  return value
55
60
  end
56
61
 
62
+ def add_default_roles
63
+ return if not @hosts.size > 0
64
+ @hosts.first[:roles] << "first"
65
+ @hosts.last[:roles] << "last"
66
+ end
67
+
57
68
  def finalize
58
69
  return if @finalized
59
70
  instance_eval &config_proc if config_proc
60
71
  fill_hosts
72
+ add_default_roles
61
73
  @hosts.freeze
62
74
  @parallel_runs.freeze
63
75
  @finalized = true
64
76
  end
65
77
 
78
+ def cleaned_roles(roles)
79
+ error "roles must be an array" if not roles.is_a? Array
80
+ roles.each do |r|
81
+ r.strip!
82
+ error "invalid role value #{r} " if not r =~ /\A[0-9A-Za-z_\-]+\z/
83
+ end
84
+ return roles
85
+ end
86
+
66
87
  end
67
88
  end
68
89
 
@@ -0,0 +1,129 @@
1
+ require 'dplyr/report'
2
+ require 'dply/logger'
3
+ require 'dplyr/remote_task'
4
+
5
+ module Dplyr
6
+ class TaskRunner
7
+
8
+ include ::Dply::Logger
9
+
10
+ attr_reader :hosts, :parallel_jobs, :task, :messages, :exit_statuses
11
+ attr_writer :auto_serialize
12
+
13
+ def initialize(hosts, task, parallel_jobs: 1)
14
+ @hosts = hosts
15
+ @parallel_jobs = parallel_jobs
16
+ @task = task
17
+
18
+ @messages = {}
19
+ @exit_statuses = {}
20
+ @auto_serialize = true
21
+ end
22
+
23
+ def run
24
+ if parallel_jobs > 1 && hosts.count > 1
25
+ run_in_parallel
26
+ else
27
+ run_serially
28
+ end
29
+ report.print_full
30
+ end
31
+
32
+ def run_serially
33
+ hosts.each do |host|
34
+ task = execute_serially host
35
+ break if task.exit_status != 0
36
+ end
37
+ end
38
+
39
+ def run_in_parallel
40
+ if @auto_serialize
41
+ execute_serially hosts[0]
42
+ execute_in_parallel Range.new(1,hosts.size - 2)
43
+ execute_serially hosts[-1]
44
+ else
45
+ execute_in_parallel Range.new(0, hosts.size - 1)
46
+ end
47
+ end
48
+
49
+ private
50
+
51
+ def remote_task(host)
52
+ RemoteTask.new(host, @task, id_size: host_id_max_width)
53
+ end
54
+
55
+ def execute_serially(host)
56
+ task = remote_task host
57
+ task.run
58
+ @exit_statuses[host] = task.exit_status
59
+ @messages[host] = task.messages
60
+ return task
61
+ end
62
+
63
+ def execute_in_parallel(range)
64
+ init_run
65
+ queue_hosts range
66
+ parallel_jobs.times do
67
+ spawn_queued_job
68
+ end
69
+ loop do
70
+ break if @queue.empty?
71
+ @mq.pop
72
+ spawn_queued_job
73
+ end
74
+ wait_for_threads
75
+ end
76
+
77
+ def start_task_thread(host)
78
+ t = Thread.new do
79
+ Thread.current[:messages] = []
80
+ task = remote_task(host)
81
+ begin
82
+ task.run
83
+ Thread.current[:messages] = task.messages
84
+ Thread.current[:exit_status] = task.exit_status
85
+ rescue => e
86
+ puts e.message
87
+ ensure
88
+ @mq << true
89
+ end
90
+ end
91
+ t.abort_on_exception = true
92
+ t.run
93
+ @threads[host] = t
94
+ end
95
+
96
+ def init_run
97
+ @mq = Queue.new
98
+ @queue = Queue.new
99
+ @threads = {}
100
+ end
101
+
102
+ def queue_hosts(range)
103
+ range.each { |i| @queue << hosts[i] }
104
+ end
105
+
106
+ def spawn_queued_job
107
+ return if @queue.empty?
108
+ host = @queue.pop(false)
109
+ start_task_thread host
110
+ rescue ThreadError
111
+ end
112
+
113
+ def host_id_max_width
114
+ @host_id_max_width ||= hosts.map {|h| h[:id].size }.max
115
+ end
116
+
117
+ def report
118
+ @report ||= Report.new(hosts, exit_statuses, messages)
119
+ end
120
+
121
+ def wait_for_threads
122
+ @threads.each do |host, t|
123
+ @exit_statuses[host] = t.value
124
+ @messages[host] = t[:messages]
125
+ end
126
+ end
127
+
128
+ end
129
+ end
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: dply
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.1.13
4
+ version: 0.1.17
5
5
  platform: ruby
6
6
  authors:
7
7
  - Neeraj
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2015-04-13 00:00:00.000000000 Z
11
+ date: 2015-05-03 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: ruby-elf
@@ -98,9 +98,11 @@ files:
98
98
  - lib/dply/cli/build.rb
99
99
  - lib/dply/cli/depcheck.rb
100
100
  - lib/dply/cli/deploy.rb
101
+ - lib/dply/cli/devbuild.rb
101
102
  - lib/dply/cli/install_pkgs.rb
102
103
  - lib/dply/cli/reload.rb
103
104
  - lib/dply/cli/reopen_logs.rb
105
+ - lib/dply/cli/setup.rb
104
106
  - lib/dply/cli/status.rb
105
107
  - lib/dply/cli/task.rb
106
108
  - lib/dply/config.rb
@@ -128,6 +130,9 @@ files:
128
130
  - lib/dply/strategy/archive.rb
129
131
  - lib/dply/strategy/git.rb
130
132
  - lib/dply/tasks.rb
133
+ - lib/dply/templates/build.erb
134
+ - lib/dply/templates/deploy.erb
135
+ - lib/dply/templates/pkgs.erb
131
136
  - lib/dply/version.rb
132
137
  - lib/dply/yum.rb
133
138
  - lib/dplyr/cli.rb
@@ -135,6 +140,7 @@ files:
135
140
  - lib/dplyr/report.rb
136
141
  - lib/dplyr/stage.rb
137
142
  - lib/dplyr/stages_config.rb
143
+ - lib/dplyr/task_runner.rb
138
144
  - lib/dplyr/tasks_config.rb
139
145
  homepage: ''
140
146
  licenses:
@@ -156,7 +162,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
156
162
  version: '0'
157
163
  requirements: []
158
164
  rubyforge_project:
159
- rubygems_version: 2.2.2
165
+ rubygems_version: 2.4.5
160
166
  signing_key:
161
167
  specification_version: 4
162
168
  summary: rake based deploy tool