dply 0.2.19 → 0.3.0

Sign up to get free protection for your applications and to get access to all the features.
Files changed (96) hide show
  1. checksums.yaml +5 -5
  2. data/.rspec +4 -0
  3. data/Rakefile +0 -14
  4. data/TODO +0 -1
  5. data/code_dump/old_remote_task.rb +2 -0
  6. data/{dev_bin → dev_exe}/dplyr +0 -0
  7. data/{dev_bin → dev_exe}/drake +1 -1
  8. data/dply.gemspec +2 -2
  9. data/{bin → exe}/dplyr +0 -0
  10. data/{bin → exe}/drake +12 -14
  11. data/lib/dply/TEST_TODO +50 -0
  12. data/lib/dply/app_config.rb +108 -0
  13. data/lib/dply/base_config.rb +110 -0
  14. data/lib/dply/build.rb +17 -11
  15. data/lib/dply/build_config.rb +28 -96
  16. data/lib/dply/bundle.rb +7 -30
  17. data/lib/dply/cli/build.rb +5 -12
  18. data/lib/dply/cli/ctl.rb +7 -8
  19. data/lib/dply/cli/deploy.rb +6 -10
  20. data/lib/dply/cli/devbuild.rb +6 -10
  21. data/lib/dply/cli/install_pkgs.rb +2 -3
  22. data/lib/dply/cli/run.rb +27 -0
  23. data/lib/dply/cli/status.rb +1 -2
  24. data/lib/dply/cli/task.rb +6 -12
  25. data/lib/dply/code_archive.rb +123 -0
  26. data/lib/dply/command.rb +57 -0
  27. data/lib/dply/config_downloader.rb +3 -2
  28. data/lib/dply/curl.rb +1 -5
  29. data/lib/dply/custom_logger.rb +18 -1
  30. data/lib/dply/deplist.rb +16 -48
  31. data/lib/dply/deploy_config.rb +34 -0
  32. data/lib/dply/elf.rb +60 -0
  33. data/lib/dply/env.rb +9 -0
  34. data/lib/dply/git.rb +15 -8
  35. data/lib/dply/helper.rb +21 -33
  36. data/lib/dply/linker.rb +27 -27
  37. data/lib/dply/lock.rb +2 -9
  38. data/lib/dply/logger.rb +1 -1
  39. data/lib/dply/pkgs.rb +9 -11
  40. data/lib/dply/release.rb +2 -2
  41. data/lib/dply/{archive.rb → remote_archive.rb} +1 -1
  42. data/lib/dply/repo.rb +3 -3
  43. data/lib/dply/rpm.rb +12 -20
  44. data/lib/dply/scripts/depcheck.rb +4 -0
  45. data/lib/dply/shared_dirs.rb +1 -1
  46. data/lib/dply/strategy/archive.rb +15 -22
  47. data/lib/dply/strategy/base.rb +82 -0
  48. data/lib/dply/strategy/git.rb +18 -19
  49. data/lib/dply/task_dsl.rb +101 -0
  50. data/lib/dply/util.rb +75 -0
  51. data/lib/dply/venv.rb +53 -0
  52. data/lib/dply/version.rb +1 -1
  53. data/lib/dply/yum.rb +21 -31
  54. data/lib/dplyr/consul.rb +1 -1
  55. data/spec/dply/base_config_spec.rb +178 -0
  56. data/spec/dply/bundle_spec.rb +100 -0
  57. data/spec/dply/command_spec.rb +190 -0
  58. data/spec/dply/curl_spec.rb +41 -0
  59. data/spec/dply/deplist_spec.rb +48 -0
  60. data/spec/dply/elf_spec.rb +64 -0
  61. data/spec/dply/env_spec.rb +57 -0
  62. data/spec/dply/git_spec.rb +136 -0
  63. data/spec/dply/helper_spec.rb +168 -0
  64. data/spec/dply/linker_spec.rb +81 -0
  65. data/spec/dply/lock_spec.rb +24 -0
  66. data/spec/dply/pkgs_spec.rb +105 -0
  67. data/spec/dply/repo_spec.rb +58 -0
  68. data/spec/dply/rpm_spec.rb +32 -0
  69. data/spec/dply/yum_spec.rb +29 -0
  70. data/spec/integration/archive_flow_spec.rb +87 -0
  71. data/spec/integration/git_flow_spec.rb +63 -0
  72. data/spec/repo.rb +27 -0
  73. data/spec/spec_helper.rb +44 -0
  74. data/spec/test_data/build.tar.gz +0 -0
  75. data/spec/test_data/build.tar.gz.md5 +1 -0
  76. data/spec/test_data/bundle/gems_installed/Gemfile +1 -0
  77. data/spec/test_data/bundle/gems_not_installed/Gemfile +2 -0
  78. data/spec/test_data/bundle/no_gemfile/.gitkeep +0 -0
  79. data/spec/test_data/command/test.rb +7 -0
  80. data/spec/test_data/elf/elf +0 -0
  81. data/spec/test_data/elf/libpgtypes.so.3 +0 -0
  82. data/spec/test_data/elf/not_elf +1 -0
  83. data/spec/test_data/sample_repo/.dply.lock +0 -0
  84. data/spec/test_data/sample_repo/Gemfile +2 -0
  85. data/spec/test_data/sample_repo/Rakefile +3 -0
  86. data/spec/test_data/sample_repo/app.rb +1 -0
  87. data/spec/test_data/sample_repo/dply/app.rb +33 -0
  88. data/spec/test_data/sample_repo/lib/libacl.so.1 +0 -0
  89. data/spec/test_data/sample_repo/pkgs.yml +2 -0
  90. data/spec/webserver.rb +21 -0
  91. metadata +96 -28
  92. data/lib/dply/cli/app_task.rb +0 -38
  93. data/lib/dply/config.rb +0 -120
  94. data/lib/dply/config_struct.rb +0 -52
  95. data/lib/dply/rakelib/drake.rake +0 -33
  96. data/lib/dply/tasks.rb +0 -136
@@ -0,0 +1,9 @@
1
+ module Dply
2
+ module Env
3
+ def self.build_env(extra_env, bundled_env:)
4
+ original_env = defined?(Bundler) ? Bundler.original_env : ENV
5
+ base_env = bundled_env ? ENV : original_env
6
+ base_env.to_h.merge(extra_env)
7
+ end
8
+ end
9
+ end
@@ -7,16 +7,23 @@ module Dply
7
7
  def self.pull(branch)
8
8
  cmd "git fetch"
9
9
  checkout(branch)
10
- if tracking_branch = get_tracking_branch(branch)
10
+ if tracking_branch = tracking_branch(branch)
11
11
  cmd "git merge #{tracking_branch}"
12
12
  else
13
13
  cmd "git pull origin #{branch}"
14
14
  end
15
15
  end
16
-
16
+
17
17
  def self.checkout(branch)
18
- current_branch = `git rev-parse --abbrev-ref HEAD `.chomp.to_sym
19
- cmd "git checkout #{branch}" if branch != current_branch
18
+ current_branch = `git rev-parse --abbrev-ref HEAD `.chomp
19
+ return if branch == current_branch
20
+
21
+ local_branch_exists = system "git", "show-ref", "-q", "--verify", "refs/heads/#{branch}"
22
+ if local_branch_exists
23
+ cmd "git checkout #{branch} --"
24
+ else
25
+ cmd "git checkout -b #{branch} -t origin/#{branch}"
26
+ end
20
27
  end
21
28
 
22
29
  def self.clone(repo, dir, mirror: nil)
@@ -33,17 +40,17 @@ module Dply
33
40
  cmd "git clean -dxf "
34
41
  end
35
42
 
36
- def self.get_tracking_branch(branch)
43
+ def self.tracking_branch(branch)
37
44
  command = "git for-each-ref --format='%(upstream:short)' refs/heads/#{branch} --count=1"
38
- tracking_branch = `#{command}`
45
+ tracking_branch = cmd command, return_output: true, display: false
39
46
  if tracking_branch =~ /[a-zA-Z0-9_]/
40
47
  return tracking_branch.chomp!
41
48
  else
42
49
  return nil
43
- end
50
+ end
44
51
  end
45
52
 
46
- def self.get_remote_url
53
+ def self.remote_url
47
54
  remote_url = cmd "git config --get remote.origin.url", return_output: true, display: false
48
55
  logger.debug remote_url.chomp
49
56
  remote_url.chomp
@@ -1,44 +1,38 @@
1
- require 'dply/error'
2
- require 'dply/logger'
1
+ require_relative 'error'
2
+ require_relative 'logger'
3
+ require_relative 'command'
4
+ require_relative 'env'
3
5
  require 'fileutils'
4
6
  require 'tmpdir'
5
7
 
6
8
  module Dply
7
9
  module Helper
8
10
 
9
- def cmd(command, display: true, error_msg: nil, return_output: false, env:{}, shell: false)
10
- if command.is_a? Array
11
- command_arr = command
12
- command_str = command.join(" ")
13
- else
14
- command_arr = command.split
15
- command_str = command
16
- end
17
- stringify_values!(env)
18
- if display
19
- logger.bullet command_str
20
- else
21
- logger.debug command_str
22
- end
11
+ def cmd(command, env: {}, bundled_env: false, return_output: false, display: :bullet)
12
+ logger.command command, mode: display
13
+ new_env = Env.build_env(env, bundled_env: bundled_env)
14
+ cmd = Command.new command, env: new_env, shell: false
15
+ return_output ? cmd.capture : cmd.run
16
+ end
23
17
 
24
- run_command = shell ? command_str : command_arr
25
- output = if return_output
26
- IO.popen(env, run_command) { |f| f.read }
27
- else
28
- system(env, *run_command, 2 => 1)
29
- end
30
- return_value = $?.exitstatus
31
- error_msg ||= "non zero exit for \"#{command_str}\""
32
- error error_msg if return_value != 0
33
- return output
18
+ def sh(command, env: {}, bundled_env: false, return_output: false, display: :bullet)
19
+ logger.command command, mode: display
20
+ new_env = Env.build_env(env, bundled_env: bundled_env)
21
+ cmd = Command.new command, env: new_env, shell: true
22
+ return_output ? cmd.capture : cmd.run
34
23
  end
35
24
 
36
25
  def symlink(src, dst)
37
26
  if File.symlink? dst
27
+ # to_s to handle pathnames
28
+ return true if File.readlink(dst) == src.to_s
38
29
  Dir.mktmpdir("sym-", "./") do |d|
39
30
  dst_tmp = "#{d}/#{File.basename dst}"
40
31
  FileUtils.ln_s src, dst_tmp
41
- cmd "mv #{dst_tmp} #{File.dirname dst}", display: false
32
+
33
+ # using 'mv' here as FileUtils.mv doesn't replace a symlink which points
34
+ # to a directory
35
+ cmd ["mv", dst_tmp, File.dirname(dst)], display: false
42
36
  end
43
37
  elsif File.exist? dst
44
38
  error "cannot create symlink #{dst} => #{src}"
@@ -47,12 +41,6 @@ module Dply
47
41
  end
48
42
  end
49
43
 
50
- def stringify_values!(hash)
51
- hash.each do |k,v|
52
- hash[k] = v.to_s
53
- end
54
- end
55
-
56
44
  def logger
57
45
  Logger.logger
58
46
  end
@@ -4,53 +4,53 @@ module Dply
4
4
 
5
5
  include Helper
6
6
 
7
- attr_reader :src_dir, :dest_dir, :map
8
-
9
7
  def initialize(src_dir, dest_dir, map: {})
10
8
  verify_absolute src_dir, dest_dir
11
9
  @src_dir = src_dir
12
10
  @dest_dir = dest_dir
13
11
  @map = map
12
+ validate_map!
14
13
  end
15
-
14
+
16
15
  def create_symlinks
17
- mapped_files.each do |f|
18
- target = link_target(f)
19
- source = link_source(f)
20
- relative_source = link_relative_source(source, target)
21
- logger.debug "linking #{target} -> #{source}"
22
- error "source #{source} doesn't exist" if not File.exist? source
23
- symlink(relative_source, target)
16
+ link_pairs.each do |src, dest|
17
+ relative_source = src_relative_to_dest(src, dest)
18
+ logger.debug "linking #{dest} -> #{src}" if logger.debug?
19
+ error "source #{src} doesn't exist" if not src.exist?
20
+ symlink(relative_source, dest)
24
21
  end
25
22
  end
26
23
 
27
- def link_target(relative_target)
28
- Pathname.new "#{dest_dir}/#{relative_target}"
29
- end
24
+ private
30
25
 
31
- def link_source(relative_target)
32
- relative_source = map[relative_target]
33
- Pathname.new "#{src_dir}/#{relative_source}"
26
+ def link_pairs
27
+ Enumerator.new do |y|
28
+ @map.each do |dest, src|
29
+ src_path = Pathname.new "#{@src_dir}/#{src}"
30
+ dest_path = Pathname.new "#{@dest_dir}/#{dest}"
31
+ y.yield [src_path, dest_path]
32
+ end
33
+ end
34
34
  end
35
35
 
36
- def link_relative_source(source, target)
37
- source.relative_path_from target.parent
38
- end
39
-
40
- def mapped_files
41
- map.keys.collect do |k|
42
- path = Pathname.new k
43
- raise "config map path cannot be absoulte" if path.absolute?
44
- k
45
- end
36
+ def src_relative_to_dest(src, dest)
37
+ src.relative_path_from dest.parent
46
38
  end
47
39
 
48
40
  def verify_absolute(*paths)
49
41
  paths.each do |path|
50
42
  absolute = Pathname.new(path).absolute?
51
- raise "#{path} not absolute" if not absolute
43
+ raise Error, "#{path} not absolute" if not absolute
52
44
  end
53
45
  end
54
46
 
47
+ def validate_map!
48
+ @map.each do |dest, src|
49
+ dest_path = Pathname.new dest
50
+ src_path = Pathname.new src
51
+ raise Error, "dest path #{dest_path} not relative" if not dest_path.relative?
52
+ raise Error, "src path #{src_path} not relative" if not src_path.relative?
53
+ end
54
+ end
55
55
  end
56
56
  end
@@ -4,7 +4,7 @@ module Dply
4
4
 
5
5
  include Helper
6
6
 
7
- def initialize(dir = nil)
7
+ def initialize(dir)
8
8
  @dir = dir
9
9
  end
10
10
 
@@ -15,14 +15,7 @@ module Dply
15
15
  end
16
16
 
17
17
  def lock_file
18
- @lock_file ||= File.open("#{dir}/.dply.lock", "a+")
18
+ @lock_file ||= File.open("#{@dir}/.dply.lock", "a+")
19
19
  end
20
-
21
- private
22
-
23
- def dir
24
- @dir ||= Dir.pwd
25
- end
26
-
27
20
  end
28
21
  end
@@ -1,5 +1,5 @@
1
1
  require 'logger'
2
- require 'dply/custom_logger'
2
+ require_relative 'custom_logger'
3
3
  module Dply
4
4
  module Logger
5
5
 
@@ -1,5 +1,5 @@
1
1
  require 'yaml'
2
- require 'dply/helper'
2
+ require_relative 'helper'
3
3
  require_relative 'yum'
4
4
 
5
5
  module Dply
@@ -9,19 +9,19 @@ module Dply
9
9
 
10
10
  attr_reader :runtime, :build, :all
11
11
 
12
- def initialize(pkgs_yml = nil)
13
- @pkgs_yml = pkgs_yml || "pkgs.yml"
12
+ def initialize(pkgs_yml)
13
+ @pkgs_yml = pkgs_yml
14
14
  read_config
15
15
  end
16
16
 
17
- def install(build_mode: false, sudo: false)
17
+ def install(build_mode: false)
18
18
  pkgs = build_mode ? @all : @runtime
19
- Yum.new(pkgs, sudo: sudo).install
19
+ Yum.install pkgs
20
20
  end
21
21
 
22
22
  def installed?(build_mode: false)
23
23
  pkgs = build_mode ? @all : @runtime
24
- Yum.new(pkgs).installed?
24
+ Yum.installed? pkgs
25
25
  end
26
26
 
27
27
  private
@@ -44,18 +44,16 @@ module Dply
44
44
  return {}
45
45
  end
46
46
  YAML.safe_load(File.read(@pkgs_yml)) || {}
47
- rescue => e
48
- error "error loading pkgs list"
47
+ rescue
48
+ error "error loading pkgs list"
49
49
  end
50
50
 
51
51
  def validate!(pkg)
52
52
  msg = "invalid pkg name #{pkg}"
53
53
  error msg if pkg =~ /\.rpm\z/i
54
- error msg if pkg =~/\A[A-Za-z_0-9\.\-]\z/
54
+ error msg if not pkg =~ /\A[A-Za-z_0-9\.\-]+\z/
55
55
  return true
56
56
  end
57
57
 
58
58
  end
59
59
  end
60
-
61
-
@@ -1,4 +1,4 @@
1
- require 'dply/archive'
1
+ require_relative 'remote_archive'
2
2
  require 'dply/helper'
3
3
  require 'tmpdir'
4
4
 
@@ -74,7 +74,7 @@ module Dply
74
74
  private
75
75
 
76
76
  def archive
77
- @archive ||= Archive.new(url, verify_checksum: @verify_checksum)
77
+ @archive ||= RemoteArchive.new(url, verify_checksum: @verify_checksum)
78
78
  end
79
79
 
80
80
  def timestamp
@@ -4,7 +4,7 @@ require 'uri'
4
4
  require 'tmpdir'
5
5
 
6
6
  module Dply
7
- class Archive
7
+ class RemoteArchive
8
8
 
9
9
  include Helper
10
10
 
@@ -12,7 +12,7 @@ module Dply
12
12
 
13
13
  def create
14
14
  if Dir.exist? "#{dir}/.git"
15
- raise "unable to create repo" if not verify_remote_url
15
+ raise Error, "unable to create repo (another repo already exists)" if not verify_remote_url
16
16
  else
17
17
  Git.clone upstream, dir, mirror: @mirror
18
18
  end
@@ -22,10 +22,10 @@ module Dply
22
22
 
23
23
  def verify_remote_url
24
24
  remote_url = Dir.chdir(dir) do
25
- Git.get_remote_url
25
+ Git.remote_url
26
26
  end
27
27
  remote_url == upstream
28
28
  end
29
-
29
+
30
30
  end
31
31
  end
@@ -1,32 +1,24 @@
1
- require 'dply/helper'
1
+ require_relative 'helper'
2
2
 
3
3
  module Dply
4
- class Rpm
5
- include Helper
4
+ module Rpm
5
+ extend Helper
6
6
 
7
7
  # libs should include ()(64bit) in their name if 64bit libs
8
-
9
- def libs_packages_map(libs)
8
+ def self.libs_pkgs_map(libs)
10
9
  h = {}
11
- libs.each do |lib|
12
- list = whatprovides(lib)
13
- h[lib] = list if not list.empty?
14
- end
10
+ libs.each { |lib| h[lib] = pkg_list(lib) }
15
11
  return h
16
12
  end
17
13
 
18
- private
19
-
20
- def filtered?(pkg)
21
- @filtered ||= ["glibc", "libgcc", "libstdc++", "openssl", "ruby-alt", "jemalloc"]
22
- @filtered.include? pkg.strip
23
- end
14
+ class << self
15
+ private
24
16
 
25
- def whatprovides(lib)
26
- command = ["rpm", "--queryformat", "%{NAME} ", "-q", "--whatprovides", lib]
27
- output = cmd command, return_output: true, display: false
28
- list = output.strip.split.select {|pkg| not filtered? pkg }
17
+ def pkg_list(lib)
18
+ command = ["rpm", "--queryformat", "%{NAME} ", "-q", "--whatprovides", lib]
19
+ output = cmd command, return_output: true, display: false
20
+ output.strip.split
21
+ end
29
22
  end
30
-
31
23
  end
32
24
  end
@@ -0,0 +1,4 @@
1
+ require_relative '../deplist'
2
+
3
+ deplist = Dply::Deplist.new(ARGV.shift)
4
+ deplist.verify!
@@ -9,7 +9,7 @@ module Dply
9
9
 
10
10
  def create
11
11
  FileUtils.mkdir_p dirs
12
- end
12
+ end
13
13
 
14
14
  def create_in(dir)
15
15
  Dir.chdir(dir) { create }
@@ -1,5 +1,5 @@
1
1
  require 'dply/helper'
2
- require 'dply/tasks'
2
+ require 'dply/util'
3
3
  require 'dply/setup'
4
4
  require 'dply/config_downloader'
5
5
  require 'dply/yum'
@@ -7,11 +7,10 @@ require 'dply/release'
7
7
  require 'dply/release_helper'
8
8
  require 'forwardable'
9
9
 
10
-
11
10
  module Dply
12
11
  module Strategy
13
12
  class Archive
14
-
13
+
15
14
  extend Forwardable
16
15
  include Helper
17
16
 
@@ -19,20 +18,20 @@ module Dply
19
18
  def_delegators :config, :target, :branch, :revision, :name,
20
19
  :config_map, :dir_map, :config_skip_download,
21
20
  :config_download_url, :build_url
22
-
21
+
23
22
  def initialize(config, options)
24
23
  @config = config
25
24
  @options = options || {}
26
25
  end
27
26
 
28
27
  def deploy
29
- error "revision not specified( use -r)" if not revision
28
+ error "revision not specified( use -r)" if not revision
30
29
  setup.archive
31
30
  if release.already_deployed? && release.current?
32
31
  logger.info "revision #{revision} is currently deployed"
33
32
  current_version = previous_version = get_release
34
33
  prune_releases
35
- tasks.report_changes(current_version, previous_version)
34
+ util.report_changes(current_version, previous_version)
36
35
  return
37
36
  end
38
37
  download_configs if config_download_url
@@ -40,25 +39,19 @@ module Dply
40
39
  previous_version = get_release
41
40
  release.make_current
42
41
  Dir.chdir current_dir do
43
- tasks.deploy deploy_target
42
+ util.run "deploy:#{deploy_target}"
44
43
  end
45
44
  release.record_deployment
46
45
  current_version = get_release
47
46
  prune_releases
48
- tasks.report_changes(previous_version, current_version)
47
+ util.report_changes(previous_version, current_version)
49
48
  end
50
-
49
+
51
50
  def reload
52
51
  download_configs if config_download_url
53
52
  Dir.chdir current_dir do
54
53
  link_all
55
- tasks.reload
56
- end
57
- end
58
-
59
- def task(name)
60
- Dir.chdir current_dir do
61
- tasks.rake name
54
+ util.run :reload
62
55
  end
63
56
  end
64
57
 
@@ -80,7 +73,7 @@ module Dply
80
73
  def download_configs
81
74
  files = config_map.values.uniq
82
75
  downloader = ConfigDownloader.new(files, config_download_url)
83
- downloader.skip_download = config_skip_download if config_skip_download
76
+ downloader.skip_download = config_skip_download if config_skip_download
84
77
  downloader.download_all
85
78
  end
86
79
 
@@ -98,21 +91,21 @@ module Dply
98
91
  release.install
99
92
  Dir.chdir release.path do
100
93
  link_all
101
- tasks.install_pkgs(use_yum: options[:use_yum])
94
+ util.install_pkgs
102
95
  end
103
96
  end
104
97
 
105
98
  def link_all
106
- tasks.link "#{config.dir}/shared", dir_map
107
- tasks.link "#{config.dir}/config", config_map
99
+ util.link "#{config.dir}/shared", dir_map
100
+ util.link "#{config.dir}/config", config_map
108
101
  end
109
102
 
110
103
  def setup
111
104
  @setup ||= Setup.new(@config)
112
105
  end
113
106
 
114
- def tasks
115
- @tasks ||= Tasks.new
107
+ def util
108
+ @util ||= Util.new
116
109
  end
117
110
 
118
111
  def prune_releases