dply 0.0.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/.gitignore +54 -0
- data/Gemfile +4 -0
- data/LICENSE.txt +22 -0
- data/README.md +34 -0
- data/Rakefile +2 -0
- data/bin/dplyr +76 -0
- data/bin/drake +84 -0
- data/code_dump/alt_remote_task.rb +119 -0
- data/code_dump/old_remote_task.rb +146 -0
- data/dply.gemspec +22 -0
- data/lib/dply.rb +5 -0
- data/lib/dply/cli/deploy.rb +69 -0
- data/lib/dply/config.rb +102 -0
- data/lib/dply/custom_logger.rb +29 -0
- data/lib/dply/deploy.rb +33 -0
- data/lib/dply/dplyr.rb +45 -0
- data/lib/dply/error.rb +4 -0
- data/lib/dply/ext/string.rb +33 -0
- data/lib/dply/git.rb +43 -0
- data/lib/dply/helper.rb +35 -0
- data/lib/dply/linker.rb +71 -0
- data/lib/dply/lock.rb +24 -0
- data/lib/dply/logger.rb +27 -0
- data/lib/dply/release.rb +34 -0
- data/lib/dply/remote_task.rb +139 -0
- data/lib/dply/repo.rb +31 -0
- data/lib/dply/report.rb +65 -0
- data/lib/dply/setup.rb +12 -0
- data/lib/dply/setup/default.rb +31 -0
- data/lib/dply/setup/git.rb +52 -0
- data/lib/dply/setup/release.rb +58 -0
- data/lib/dply/shared_dirs.rb +28 -0
- data/lib/dply/shell.rb +47 -0
- data/lib/dply/stages_config.rb +94 -0
- data/lib/dply/strategy.rb +12 -0
- data/lib/dply/strategy/default.rb +50 -0
- data/lib/dply/strategy/git.rb +85 -0
- data/lib/dply/tasks.rb +37 -0
- data/lib/dply/version.rb +3 -0
- metadata +112 -0
data/lib/dply/lock.rb
ADDED
@@ -0,0 +1,24 @@
|
|
1
|
+
require 'dply/helper'
|
2
|
+
module Dply
|
3
|
+
class Lock
|
4
|
+
|
5
|
+
include Helper
|
6
|
+
attr_accessor :deploy_dir
|
7
|
+
|
8
|
+
def initialize(deploy_dir)
|
9
|
+
@deploy_dir = deploy_dir
|
10
|
+
end
|
11
|
+
|
12
|
+
def lock_file
|
13
|
+
@lock_file ||= Dir.chdir(deploy_dir) do
|
14
|
+
File.open(".dply.lock", "w+")
|
15
|
+
end
|
16
|
+
end
|
17
|
+
|
18
|
+
def acquire
|
19
|
+
logger.debug "acquiring exclusive lock"
|
20
|
+
lock_file.flock(File::LOCK_EX)
|
21
|
+
end
|
22
|
+
|
23
|
+
end
|
24
|
+
end
|
data/lib/dply/logger.rb
ADDED
@@ -0,0 +1,27 @@
|
|
1
|
+
require 'logger'
|
2
|
+
require 'dply/custom_logger'
|
3
|
+
module Dply
|
4
|
+
module Logger
|
5
|
+
|
6
|
+
def self.logger
|
7
|
+
@logger ||= ::Dply::CustomLogger.new(STDOUT)
|
8
|
+
end
|
9
|
+
|
10
|
+
def self.stderr
|
11
|
+
@stderr ||= ::Logger.new(STDERR)
|
12
|
+
end
|
13
|
+
|
14
|
+
def logger
|
15
|
+
::Dply::Logger.logger
|
16
|
+
end
|
17
|
+
|
18
|
+
def stderr
|
19
|
+
::Dply::Logger.stderr
|
20
|
+
end
|
21
|
+
|
22
|
+
def debug?
|
23
|
+
logger.level == ::Logger::DEBUG
|
24
|
+
end
|
25
|
+
|
26
|
+
end
|
27
|
+
end
|
data/lib/dply/release.rb
ADDED
@@ -0,0 +1,34 @@
|
|
1
|
+
module Dply
|
2
|
+
class Release
|
3
|
+
|
4
|
+
attr_reader release_dir
|
5
|
+
|
6
|
+
def initialize(release_dir)
|
7
|
+
@release_dir = release_dir
|
8
|
+
end
|
9
|
+
|
10
|
+
def create(repo_cache, branch)
|
11
|
+
return if exists?
|
12
|
+
FileUtils.mkdir uncommited_release_dir
|
13
|
+
copy_source_code(repo_cache, branch)
|
14
|
+
end
|
15
|
+
|
16
|
+
def copy_source_code(repo_cache, branch)
|
17
|
+
cmd "git archive #{branch} | tar -x -f - -C #{uncommited_release_dir}"
|
18
|
+
end
|
19
|
+
|
20
|
+
def exists?
|
21
|
+
File.exists? (release_dir) || File.exists? (uncommited_release_dir)
|
22
|
+
end
|
23
|
+
|
24
|
+
def uncommited_release_dir
|
25
|
+
"#{release_dir}.uncommited"
|
26
|
+
end
|
27
|
+
|
28
|
+
def commit
|
29
|
+
return if File.exists? (release_dir)
|
30
|
+
FileUtils.mv uncommited_release_dir, release_dir
|
31
|
+
end
|
32
|
+
|
33
|
+
end
|
34
|
+
end
|
@@ -0,0 +1,139 @@
|
|
1
|
+
require 'pty'
|
2
|
+
require 'dply/report'
|
3
|
+
require 'dply/logger'
|
4
|
+
|
5
|
+
module Dply
|
6
|
+
class RemoteTask
|
7
|
+
|
8
|
+
include Logger
|
9
|
+
|
10
|
+
attr_reader :hosts, :parallel_jobs, :task
|
11
|
+
|
12
|
+
def initialize(hosts, task, parallel_jobs: 1)
|
13
|
+
@hosts = hosts
|
14
|
+
@parallel_jobs = parallel_jobs
|
15
|
+
@task = task
|
16
|
+
end
|
17
|
+
|
18
|
+
def run
|
19
|
+
if parallel_jobs > 1 && hosts.count > 1
|
20
|
+
run_in_parallel
|
21
|
+
else
|
22
|
+
run_serially
|
23
|
+
end
|
24
|
+
end
|
25
|
+
|
26
|
+
def run_in_parallel
|
27
|
+
init_run
|
28
|
+
parallel_jobs.times do
|
29
|
+
spawn_queued_job
|
30
|
+
end
|
31
|
+
loop do
|
32
|
+
break if @queue.empty?
|
33
|
+
@mq.pop
|
34
|
+
spawn_queued_job
|
35
|
+
end
|
36
|
+
report.print_full
|
37
|
+
end
|
38
|
+
|
39
|
+
def run_serially
|
40
|
+
hosts.each do |host_info|
|
41
|
+
puts "=== Running on #{host_info[:id]} ==="
|
42
|
+
run_cmd = remote_cmd host_info
|
43
|
+
system run_cmd
|
44
|
+
puts
|
45
|
+
raise ::Dply::Error, "remote deploy failed on #{host_info[:id]}" if $? != 0
|
46
|
+
end
|
47
|
+
end
|
48
|
+
|
49
|
+
private
|
50
|
+
|
51
|
+
def init_run
|
52
|
+
queue_all_hosts
|
53
|
+
job_output_template
|
54
|
+
@threads = {}
|
55
|
+
@mq = Queue.new
|
56
|
+
end
|
57
|
+
|
58
|
+
def queue_all_hosts
|
59
|
+
@queue = Queue.new
|
60
|
+
hosts.each { |h| @queue << h }
|
61
|
+
end
|
62
|
+
|
63
|
+
def spawn_job(host_info)
|
64
|
+
run_cmd = remote_cmd host_info
|
65
|
+
thread = popen(run_cmd, host_info)
|
66
|
+
@threads[host_info] = thread
|
67
|
+
end
|
68
|
+
|
69
|
+
def spawn_queued_job
|
70
|
+
return if @queue.empty?
|
71
|
+
host = @queue.pop(false)
|
72
|
+
spawn_job host
|
73
|
+
rescue ThreadError
|
74
|
+
end
|
75
|
+
|
76
|
+
def remote_cmd(host_info)
|
77
|
+
user = host_info[:user]
|
78
|
+
host = host_info[:host]
|
79
|
+
deploy_dir = host_info[:deploy_dir]
|
80
|
+
if logger.debug?
|
81
|
+
%(ssh -tt -oBatchMode=yes -l #{user} #{host} "drake --debug -d #{deploy_dir} #{task} 2>&1")
|
82
|
+
else
|
83
|
+
%(ssh -tt -oBatchMode=yes -l #{user} #{host} "drake -d #{deploy_dir} #{task} 2>&1" 2>/dev/null)
|
84
|
+
end
|
85
|
+
end
|
86
|
+
|
87
|
+
def pty_read(file, id)
|
88
|
+
file.each do |line|
|
89
|
+
printf @job_output_template, id, line
|
90
|
+
end
|
91
|
+
rescue EOFError,Errno::ECONNRESET, Errno::EPIPE, Errno::EIO => e
|
92
|
+
end
|
93
|
+
|
94
|
+
def popen(cmd, host_info)
|
95
|
+
t = Thread.new do
|
96
|
+
begin
|
97
|
+
r, w, pid = PTY.spawn(cmd)
|
98
|
+
pty_read(r, host_info[:id])
|
99
|
+
exit_status pid
|
100
|
+
ensure
|
101
|
+
@mq << true
|
102
|
+
end
|
103
|
+
end
|
104
|
+
t.abort_on_exception = true
|
105
|
+
t.run
|
106
|
+
return t
|
107
|
+
end
|
108
|
+
|
109
|
+
def host_id_max_width
|
110
|
+
hosts.map {|h| h[:id].size }.max
|
111
|
+
end
|
112
|
+
|
113
|
+
def job_output_template
|
114
|
+
@job_output_template ||= begin
|
115
|
+
id_template = "%-#{host_id_max_width}s".bold.grey
|
116
|
+
template = "#{id_template} %s"
|
117
|
+
end
|
118
|
+
end
|
119
|
+
|
120
|
+
def report
|
121
|
+
@report ||= Report.new(hosts, exit_statuses)
|
122
|
+
end
|
123
|
+
|
124
|
+
def exit_statuses
|
125
|
+
return @exit_statuses if @exit_statuses
|
126
|
+
@exit_statuses = {}
|
127
|
+
@threads.each do |host, thread|
|
128
|
+
@exit_statuses[host] = thread.value
|
129
|
+
end
|
130
|
+
@exit_statuses
|
131
|
+
end
|
132
|
+
|
133
|
+
def exit_status(pid)
|
134
|
+
pid, status = Process.waitpid2(pid)
|
135
|
+
return status
|
136
|
+
end
|
137
|
+
|
138
|
+
end
|
139
|
+
end
|
data/lib/dply/repo.rb
ADDED
@@ -0,0 +1,31 @@
|
|
1
|
+
require 'dply/git'
|
2
|
+
module Dply
|
3
|
+
class Repo
|
4
|
+
|
5
|
+
attr_reader :dir, :upstream, :mirror
|
6
|
+
|
7
|
+
def initialize(dir, upstream, mirror: nil)
|
8
|
+
@dir = dir
|
9
|
+
@upstream = upstream
|
10
|
+
@mirror = mirror
|
11
|
+
end
|
12
|
+
|
13
|
+
def create
|
14
|
+
if Dir.exist? "#{dir}/.git"
|
15
|
+
raise "unable to create repo" if not verify_remote_url
|
16
|
+
else
|
17
|
+
Git.clone upstream, dir
|
18
|
+
end
|
19
|
+
end
|
20
|
+
|
21
|
+
private
|
22
|
+
|
23
|
+
def verify_remote_url
|
24
|
+
remote_url = Dir.chdir(dir) do
|
25
|
+
Git.get_remote_url
|
26
|
+
end
|
27
|
+
remote_url == upstream
|
28
|
+
end
|
29
|
+
|
30
|
+
end
|
31
|
+
end
|
data/lib/dply/report.rb
ADDED
@@ -0,0 +1,65 @@
|
|
1
|
+
module Dply
|
2
|
+
class Report
|
3
|
+
|
4
|
+
attr_reader :hosts, :exit_statuses
|
5
|
+
|
6
|
+
def initialize(hosts, exit_statuses)
|
7
|
+
@hosts = hosts
|
8
|
+
@exit_statuses = exit_statuses
|
9
|
+
end
|
10
|
+
|
11
|
+
|
12
|
+
def print_full
|
13
|
+
print_successful_jobs
|
14
|
+
print_failed_jobs
|
15
|
+
print_summary
|
16
|
+
end
|
17
|
+
|
18
|
+
def succeeded
|
19
|
+
@succeeded ||= exit_statuses.keys.select do |k|
|
20
|
+
exit_statuses[k] == 0
|
21
|
+
end
|
22
|
+
end
|
23
|
+
|
24
|
+
def failed
|
25
|
+
@failed ||= exit_statuses.keys.select do |k|
|
26
|
+
exit_statuses[k] != 0
|
27
|
+
end
|
28
|
+
end
|
29
|
+
|
30
|
+
def print_successful_jobs
|
31
|
+
puts "succeeded".green
|
32
|
+
succeeded.each do |host|
|
33
|
+
puts " - #{host[:id]}"
|
34
|
+
end
|
35
|
+
end
|
36
|
+
|
37
|
+
def print_failed_jobs
|
38
|
+
return if failed.count == 0
|
39
|
+
puts "failed".red
|
40
|
+
failed.each do |host|
|
41
|
+
puts " - #{host[:id]}"
|
42
|
+
end
|
43
|
+
end
|
44
|
+
|
45
|
+
|
46
|
+
def print_summary
|
47
|
+
total_hosts = hosts.count
|
48
|
+
run_count = @exit_statuses.count
|
49
|
+
not_run = total_hosts - run_count
|
50
|
+
if (failed.count > 0 || not_run > 0 )
|
51
|
+
not_run_error = "not run on #{not_run}/#{total_hosts}" if not_run > 0
|
52
|
+
failed_error = "failed on #{failed.count} of #{total_hosts} hosts" if failed.count > 0
|
53
|
+
|
54
|
+
errors = []
|
55
|
+
errors << not_run_error if not_run_error
|
56
|
+
errors << failed_error if failed_error
|
57
|
+
|
58
|
+
error_str = "task #{errors.join(", ")}"
|
59
|
+
raise ::Dply::Error, error_str
|
60
|
+
end
|
61
|
+
puts "tasks ran successfully on #{succeeded.count}/#{total_hosts} hosts"
|
62
|
+
end
|
63
|
+
|
64
|
+
end
|
65
|
+
end
|
data/lib/dply/setup.rb
ADDED
@@ -0,0 +1,31 @@
|
|
1
|
+
require 'fileutils'
|
2
|
+
require 'dply/helper'
|
3
|
+
require 'dply/shared_dirs'
|
4
|
+
|
5
|
+
module Dply
|
6
|
+
module Setup
|
7
|
+
class Default
|
8
|
+
|
9
|
+
include Helper
|
10
|
+
|
11
|
+
attr_reader :config
|
12
|
+
|
13
|
+
def initialize(config)
|
14
|
+
@config = config
|
15
|
+
end
|
16
|
+
|
17
|
+
def run
|
18
|
+
Dir.chdir config.deploy_dir do
|
19
|
+
shared_dirs.create
|
20
|
+
end
|
21
|
+
end
|
22
|
+
|
23
|
+
private
|
24
|
+
|
25
|
+
def shared_dirs
|
26
|
+
@shared_dirs ||= SharedDirs.new
|
27
|
+
end
|
28
|
+
|
29
|
+
end
|
30
|
+
end
|
31
|
+
end
|
@@ -0,0 +1,52 @@
|
|
1
|
+
require 'fileutils'
|
2
|
+
require 'dply/helper'
|
3
|
+
require 'dply/repo'
|
4
|
+
require 'dply/shared_dirs'
|
5
|
+
|
6
|
+
module Dply
|
7
|
+
module Setup
|
8
|
+
class Git
|
9
|
+
|
10
|
+
include Helper
|
11
|
+
|
12
|
+
attr_reader :config
|
13
|
+
|
14
|
+
def initialize(config)
|
15
|
+
@config = config
|
16
|
+
end
|
17
|
+
|
18
|
+
def run
|
19
|
+
Dir.chdir setup_dir do
|
20
|
+
repo.create
|
21
|
+
symlink "repo", "current"
|
22
|
+
create_dirs
|
23
|
+
shared_dirs.create_in "shared"
|
24
|
+
end
|
25
|
+
end
|
26
|
+
|
27
|
+
private
|
28
|
+
|
29
|
+
def repo
|
30
|
+
@repo ||= ::Dply::Repo.new(repo_dir, config.repo)
|
31
|
+
end
|
32
|
+
|
33
|
+
def create_dirs
|
34
|
+
dirs = ["config", "shared"]
|
35
|
+
FileUtils.mkdir_p dirs
|
36
|
+
end
|
37
|
+
|
38
|
+
def shared_dirs
|
39
|
+
@shared_dirs ||= SharedDirs.new(config.shared_dirs)
|
40
|
+
end
|
41
|
+
|
42
|
+
def setup_dir
|
43
|
+
config.deploy_dir
|
44
|
+
end
|
45
|
+
|
46
|
+
def repo_dir
|
47
|
+
"#{setup_dir}/repo"
|
48
|
+
end
|
49
|
+
|
50
|
+
end
|
51
|
+
end
|
52
|
+
end
|
@@ -0,0 +1,58 @@
|
|
1
|
+
require 'fileutils'
|
2
|
+
require 'dply/helper'
|
3
|
+
|
4
|
+
module Dply
|
5
|
+
class Setup
|
6
|
+
|
7
|
+
include Helper
|
8
|
+
|
9
|
+
attr_reader :config
|
10
|
+
|
11
|
+
def initialize(config)
|
12
|
+
@config = config
|
13
|
+
end
|
14
|
+
|
15
|
+
def run
|
16
|
+
method = "#{config.mode}_mode".to_sym
|
17
|
+
send method
|
18
|
+
end
|
19
|
+
|
20
|
+
def git_mode
|
21
|
+
Dir.chdir deploy_dir do
|
22
|
+
git.clone config.repo, "repo"
|
23
|
+
FileUtils.mkdir_p "config"
|
24
|
+
symlink "repo", "current"
|
25
|
+
end
|
26
|
+
end
|
27
|
+
|
28
|
+
def release_mode
|
29
|
+
Dir.chdir deploy_dir do
|
30
|
+
git.clone "repo_cache"
|
31
|
+
FileUtils.mkdir_p "shared", "releases", "config"
|
32
|
+
create_tmp_dirs "shared"
|
33
|
+
create_extra_shared_dirs "shared"
|
34
|
+
end
|
35
|
+
end
|
36
|
+
|
37
|
+
def default_mode
|
38
|
+
create_tmp_dirs deploy_dir
|
39
|
+
end
|
40
|
+
|
41
|
+
private
|
42
|
+
|
43
|
+
def create_tmp_dirs(dir)
|
44
|
+
dirs = "tmp/sockets", "tmp/pids", "log"
|
45
|
+
Dir.chdir(dir) { FileUtils.mkdir_p dirs }
|
46
|
+
end
|
47
|
+
|
48
|
+
def create_extra_shared_dirs(dir)
|
49
|
+
dirs = config.shared_dirs
|
50
|
+
Dir.chdir dir { FileUtils.mkdir_p dirs }
|
51
|
+
end
|
52
|
+
|
53
|
+
def deploy_dir
|
54
|
+
config.deploy_dir
|
55
|
+
end
|
56
|
+
|
57
|
+
end
|
58
|
+
end
|