kameleon-builder 2.1.3 → 2.2.0

Sign up to get free protection for your applications and to get access to all the features.
Files changed (185) hide show
  1. data/.editorconfig +0 -0
  2. data/.env +2 -6
  3. data/CHANGELOG.rst +74 -1
  4. data/Gemfile +20 -2
  5. data/README.rst +35 -25
  6. data/Thorfile +29 -0
  7. data/contrib/kameleon_bashrc.sh +61 -0
  8. data/contrib/polipo_env.sh +3 -2
  9. data/kameleon-builder.gemspec +3 -10
  10. data/lib/kameleon.rb +10 -14
  11. data/lib/kameleon/cli.rb +92 -105
  12. data/lib/kameleon/context.rb +76 -43
  13. data/lib/kameleon/engine.rb +147 -103
  14. data/lib/kameleon/environment.rb +10 -5
  15. data/lib/kameleon/error.rb +1 -32
  16. data/lib/kameleon/persistent_cache.rb +127 -29
  17. data/lib/kameleon/recipe.rb +63 -106
  18. data/lib/kameleon/shell.rb +32 -17
  19. data/lib/kameleon/step.rb +18 -1
  20. data/lib/kameleon/ui.rb +141 -0
  21. data/lib/kameleon/utils.rb +9 -0
  22. data/templates/{debian7-chroot.yaml → chroot/debian7-amd64.yaml} +29 -21
  23. data/templates/{docker-debian7.yaml → docker/debian7.yaml} +5 -5
  24. data/templates/extend.erb +2 -2
  25. data/templates/{debian7-g5k.yaml → grid5000/debian7.yaml} +1 -1
  26. data/templates/{archlinux-desktop.yaml → qemu/archlinux-desktop-i686.yaml} +2 -2
  27. data/templates/qemu/archlinux-desktop-x86_64.yaml +25 -0
  28. data/templates/qemu/archlinux-i686.yaml +25 -0
  29. data/templates/{archlinux.yaml → qemu/archlinux-x86_64.yaml} +37 -34
  30. data/templates/qemu/centos6.5-x86_64.yaml +113 -0
  31. data/templates/qemu/centos7-x86_64.yaml +119 -0
  32. data/templates/{debian7.yaml → qemu/debian7-amd64.yaml} +45 -40
  33. data/templates/{debian7-desktop.yaml → qemu/debian7-desktop-amd64.yaml} +2 -3
  34. data/templates/{debian7-kameleon.yaml → qemu/debian7-kameleon-amd64.yaml} +3 -6
  35. data/templates/{debian7-oar-dev.yaml → qemu/debian7-oar-amd64.yaml} +2 -2
  36. data/templates/qemu/debian8-amd64.yaml +25 -0
  37. data/templates/{fedora20-desktop.yaml → qemu/fedora20-desktop-x86_64.yaml} +2 -2
  38. data/templates/qemu/fedora20-x86_64.yaml +116 -0
  39. data/templates/{ubuntu-12.04.yaml → qemu/ubuntu-12.04-amd64.yaml} +42 -38
  40. data/templates/{ubuntu-12.04-desktop.yaml → qemu/ubuntu-12.04-desktop-amd64.yaml} +3 -3
  41. data/templates/{ubuntu-14.04.yaml → qemu/ubuntu-14.04-amd64.yaml} +2 -2
  42. data/templates/{ubuntu-14.04-desktop.yaml → qemu/ubuntu-14.04-desktop-amd64.yaml} +2 -2
  43. data/templates/steps/aliases/defaults.yaml +19 -13
  44. data/templates/steps/bootstrap/archlinux/arch_bootstrap.yaml +11 -19
  45. data/templates/steps/bootstrap/debian/debootstrap.yaml +10 -11
  46. data/templates/steps/bootstrap/debian/debootstrap_arm.yaml +11 -19
  47. data/templates/steps/bootstrap/fedora/init_pxeboot.yaml +13 -0
  48. data/templates/steps/bootstrap/fedora/yum_bootstrap.yaml +45 -0
  49. data/templates/steps/bootstrap/initialize_disk.yaml +16 -0
  50. data/templates/steps/bootstrap/initialize_disk_chroot.yaml +10 -14
  51. data/templates/steps/bootstrap/install_requirements.yaml +3 -0
  52. data/templates/steps/bootstrap/prepare_docker.yaml +7 -7
  53. data/templates/steps/bootstrap/prepare_qemu.yaml +20 -26
  54. data/templates/steps/bootstrap/prepare_virtualbox.yaml +63 -0
  55. data/templates/steps/bootstrap/start_chroot.yaml +2 -13
  56. data/templates/steps/bootstrap/start_qemu.yaml +50 -53
  57. data/templates/steps/bootstrap/start_virtualbox.yaml +37 -0
  58. data/templates/steps/bootstrap/switch_context_qemu.yaml +64 -0
  59. data/templates/steps/bootstrap/switch_context_virtualbox.yaml +66 -0
  60. data/templates/steps/breakpoint.yaml +2 -1
  61. data/templates/steps/checkpoints/docker.yaml +14 -11
  62. data/templates/steps/checkpoints/qcow2.yaml +26 -24
  63. data/templates/steps/checkpoints/qemu.yaml +30 -36
  64. data/templates/steps/checkpoints/virtualbox.yaml +21 -0
  65. data/templates/steps/disable_checkpoint.yaml +3 -0
  66. data/templates/steps/enable_checkpoint.yaml +5 -0
  67. data/templates/steps/export/qemu_save_appliance.yaml +52 -0
  68. data/templates/steps/export/virtualbox_save_appliance.yaml +67 -0
  69. data/templates/steps/setup/archlinux/install_bootloader.yaml +23 -0
  70. data/templates/steps/setup/centos/6.5/configure_network.yaml +31 -0
  71. data/templates/steps/setup/centos/6.5/configure_system.yaml +27 -0
  72. data/templates/steps/setup/debian/configure_apt.yaml +1 -6
  73. data/templates/steps/setup/debian/configure_kernel.yaml +0 -5
  74. data/templates/steps/setup/debian/install_bootloader.yaml +36 -0
  75. data/templates/steps/setup/debian/setup_vagrant_box.yaml +48 -18
  76. data/templates/steps/setup/debian/upgrade_system.yaml +2 -7
  77. data/templates/steps/setup/fedora/configure_kernel.yaml +9 -0
  78. data/templates/steps/setup/fedora/configure_keyboard.yaml +12 -0
  79. data/templates/steps/setup/fedora/configure_network.yaml +9 -1
  80. data/templates/steps/setup/fedora/configure_system.yaml +20 -45
  81. data/templates/steps/setup/fedora/install_bootloader.yaml +58 -0
  82. data/templates/steps/setup/fedora/minimal_install.yaml +3 -0
  83. data/templates/steps/setup/fedora/update_system.yaml +13 -5
  84. data/templates/steps/setup/ubuntu/configure_apt.yaml +4 -9
  85. data/templates/vagrant/debian7-amd64.yaml +130 -0
  86. data/templates/virtualbox/archlinux-desktop-i686.yaml +25 -0
  87. data/templates/virtualbox/archlinux-desktop-x86_64.yaml +28 -0
  88. data/templates/virtualbox/archlinux-i686.yaml +28 -0
  89. data/templates/virtualbox/archlinux-x86_64.yaml +109 -0
  90. data/templates/virtualbox/centos6.5-i386.yaml +39 -0
  91. data/templates/virtualbox/centos6.5-x86_64.yaml +111 -0
  92. data/templates/virtualbox/centos7-x86_64.yaml +116 -0
  93. data/{docs/source/debian7.yaml → templates/virtualbox/debian7-amd64.yaml} +53 -52
  94. data/templates/virtualbox/debian7-desktop-amd64.yaml +25 -0
  95. data/templates/{fedora-rawhide.yaml → virtualbox/debian7-i386.yaml} +12 -11
  96. data/templates/virtualbox/debian7-kameleon-amd64.yaml +38 -0
  97. data/templates/virtualbox/debian7-oar-amd64.yaml +51 -0
  98. data/templates/{debian-testing.yaml → virtualbox/debian8-amd64.yaml} +3 -3
  99. data/templates/virtualbox/debian8-i386.yaml +31 -0
  100. data/templates/virtualbox/fedora20-x86_64.yaml +116 -0
  101. data/templates/virtualbox/ubuntu-12.04-amd64.yaml +128 -0
  102. data/templates/virtualbox/ubuntu-12.04-desktop-amd64.yaml +25 -0
  103. data/templates/virtualbox/ubuntu-14.04-amd64.yaml +25 -0
  104. data/templates/virtualbox/ubuntu-14.04-desktop-amd64.yaml +27 -0
  105. data/version.txt +1 -1
  106. metadata +67 -212
  107. data/Rakefile +0 -24
  108. data/docs/.gitignore +0 -1
  109. data/docs/Makefile +0 -181
  110. data/docs/README.md +0 -17
  111. data/docs/make.bat +0 -242
  112. data/docs/source/_static/.gitignore +0 -0
  113. data/docs/source/_static/centos.png +0 -0
  114. data/docs/source/_static/debian.png +0 -0
  115. data/docs/source/_static/kameleon-logo.png +0 -0
  116. data/docs/source/_static/kameleon-logo.xcf +0 -0
  117. data/docs/source/_static/kameleon-long.png +0 -0
  118. data/docs/source/_static/ubuntu.png +0 -0
  119. data/docs/source/_themes/sphinx_rtd_theme/__init__.py +0 -17
  120. data/docs/source/_themes/sphinx_rtd_theme/breadcrumbs.html +0 -19
  121. data/docs/source/_themes/sphinx_rtd_theme/footer.html +0 -32
  122. data/docs/source/_themes/sphinx_rtd_theme/layout.html +0 -160
  123. data/docs/source/_themes/sphinx_rtd_theme/layout_old.html +0 -205
  124. data/docs/source/_themes/sphinx_rtd_theme/search.html +0 -50
  125. data/docs/source/_themes/sphinx_rtd_theme/searchbox.html +0 -7
  126. data/docs/source/_themes/sphinx_rtd_theme/static/css/badge_only.css +0 -1
  127. data/docs/source/_themes/sphinx_rtd_theme/static/css/theme.css +0 -4
  128. data/docs/source/_themes/sphinx_rtd_theme/static/fonts/FontAwesome.otf +0 -0
  129. data/docs/source/_themes/sphinx_rtd_theme/static/fonts/fontawesome-webfont.eot +0 -0
  130. data/docs/source/_themes/sphinx_rtd_theme/static/fonts/fontawesome-webfont.svg +0 -414
  131. data/docs/source/_themes/sphinx_rtd_theme/static/fonts/fontawesome-webfont.ttf +0 -0
  132. data/docs/source/_themes/sphinx_rtd_theme/static/fonts/fontawesome-webfont.woff +0 -0
  133. data/docs/source/_themes/sphinx_rtd_theme/static/js/theme.js +0 -47
  134. data/docs/source/_themes/sphinx_rtd_theme/theme.conf +0 -8
  135. data/docs/source/_themes/sphinx_rtd_theme/versions.html +0 -37
  136. data/docs/source/aliases.rst +0 -31
  137. data/docs/source/atlas_debian_g5k.yaml +0 -36
  138. data/docs/source/checkpoint.rst +0 -30
  139. data/docs/source/commands.rst +0 -63
  140. data/docs/source/conf.py +0 -262
  141. data/docs/source/context.rst +0 -47
  142. data/docs/source/debian_customized.yaml +0 -28
  143. data/docs/source/debian_customized_g5k.yaml +0 -21
  144. data/docs/source/faq.rst +0 -43
  145. data/docs/source/getting_started.rst +0 -260
  146. data/docs/source/grid5000_tutorial.rst +0 -525
  147. data/docs/source/index.rst +0 -53
  148. data/docs/source/install_atlas.yaml +0 -25
  149. data/docs/source/install_hpl.yaml +0 -24
  150. data/docs/source/installation.rst +0 -51
  151. data/docs/source/persistent_cache.rst +0 -34
  152. data/docs/source/recipe.rst +0 -177
  153. data/docs/source/tau_install.yaml +0 -19
  154. data/docs/source/tau_install_g5k.yaml +0 -25
  155. data/docs/source/use_cases.rst +0 -93
  156. data/docs/source/workspace.rst +0 -13
  157. data/lib/kameleon/logger.rb +0 -56
  158. data/omnibus/.gitignore +0 -11
  159. data/omnibus/.kitchen.yml +0 -25
  160. data/omnibus/Berksfile +0 -9
  161. data/omnibus/Berksfile.lock +0 -25
  162. data/omnibus/Gemfile +0 -12
  163. data/omnibus/README.md +0 -94
  164. data/omnibus/config/projects/kameleon.rb +0 -23
  165. data/omnibus/config/software/kameleon.rb +0 -24
  166. data/omnibus/config/software/polipo.rb +0 -30
  167. data/omnibus/config/software/ruby.rb +0 -158
  168. data/omnibus/files/mac_dmg/Resources/background.png +0 -0
  169. data/omnibus/files/mac_dmg/Resources/icon.png +0 -0
  170. data/omnibus/files/mac_pkg/Resources/background.png +0 -0
  171. data/omnibus/files/mac_pkg/Resources/license.html +0 -1
  172. data/omnibus/files/mac_pkg/Resources/welcome.html +0 -9
  173. data/omnibus/omnibus.rb +0 -27
  174. data/omnibus/package-scripts/kameleon/makeselfinst +0 -27
  175. data/omnibus/package-scripts/kameleon/postrm +0 -9
  176. data/templates/fedora20.yaml +0 -105
  177. data/templates/steps/bootstrap/archlinux/install_bootloader.yaml +0 -46
  178. data/templates/steps/bootstrap/archlinux/populate_disk.yaml +0 -39
  179. data/templates/steps/bootstrap/fedora/liveos_bootstrap.yaml +0 -123
  180. data/templates/steps/bootstrap/initialize_disk_qemu.yaml +0 -72
  181. data/templates/steps/bootstrap/install_bootloader.yaml +0 -42
  182. data/templates/steps/bootstrap/ubuntu/debootstrap.yaml +0 -27
  183. data/templates/steps/export/save_appliance.yaml +0 -58
  184. data/templates/steps/export/save_vagrant_box.yaml +0 -29
  185. data/templates/vagrant-debian7.yaml +0 -31
@@ -7,30 +7,35 @@ module Kameleon
7
7
  attr_accessor :workspace
8
8
  attr_accessor :templates_path
9
9
  attr_accessor :build_path
10
- attr_accessor :log_file
10
+ attr_accessor :cache_path
11
11
  attr_accessor :debug
12
12
 
13
+ def script?
14
+ @script
15
+ end
13
16
 
14
17
  def initialize(options = {})
15
- @logger = Log4r::Logger.new("kameleon::[kameleon]")
16
18
  # symbolify commandline options
17
19
  options = options.inject({}) {|result,(key,value)| result.update({key.to_sym => value})}
18
20
  workspace = File.expand_path(Dir.pwd)
19
21
  build_path = File.expand_path(options[:build_path] || File.join(workspace, "build"))
22
+ cache_path = File.expand_path(options[:cache_path] || File.join(build_path, "cache"))
20
23
  defaults = {
21
24
  :workspace => Pathname.new(workspace),
22
25
  :templates_path => Kameleon.templates_path,
23
26
  :templates_names => Kameleon.templates_names,
24
27
  :build_path => Pathname.new(build_path),
25
- :log_file => Pathname.new(File.join(workspace, "kameleon.log"))
28
+ :cache_path => Pathname.new(cache_path),
29
+ :script => options[:script],
26
30
  }
27
31
  options = defaults.merge(options)
28
- @logger.debug("Environment initialized (#{self})")
32
+ Kameleon.ui.debug("Environment initialized (#{self})")
29
33
  # Injecting all variables of the options and assign the variables
30
34
  options.each do |key, value|
31
35
  instance_variable_set("@#{key}".to_sym, options[key])
32
- @logger.debug(" @#{key} : #{options[key]}")
36
+ Kameleon.ui.debug(" @#{key} : #{options[key]}")
33
37
  end
38
+ @debug = true if ENV['KAMELEON_DEBUG']
34
39
  end
35
40
  end
36
41
  end
@@ -1,4 +1,3 @@
1
- require 'thor/error'
2
1
 
3
2
  module Kameleon
4
3
  class Error < ::StandardError
@@ -14,6 +13,7 @@ module Kameleon
14
13
  end
15
14
  end
16
15
 
16
+ class Exit < Error; status_code(0) ; end
17
17
  class KameleonError < Error; status_code(1) ; end
18
18
  class ExecError < Error; status_code(2) ; end
19
19
  class InternalError < Error; status_code(3) ; end
@@ -23,35 +23,4 @@ module Kameleon
23
23
  class BuildError < Error; status_code(7) ; end
24
24
  class AbortError < Error; status_code(8) ; end
25
25
  class TemplateNotFound < Error; status_code(9) ; end
26
-
27
- def self.with_friendly_errors
28
- yield
29
- rescue Kameleon::Error => e
30
- e.message.split( /\r?\n/ ).each {|m| Kameleon.logger.fatal m }
31
- exit e.status_code
32
- rescue Thor::UndefinedTaskError => e
33
- $stderr << "#{e.message}\n"
34
- e.backtrace.each {|m| Kameleon.logger.debug m }
35
- exit 15
36
- rescue Thor::Error => e
37
- $stderr << "#{e.message}\n"
38
- e.backtrace.each {|m| Kameleon.logger.debug m }
39
- exit 15
40
- rescue SystemExit, Interrupt => e
41
- Kameleon.logger.fatal("Quitting...")
42
- exit 1
43
- rescue Errno::ENOENT => e
44
- $stderr << "#{e.message}\n"
45
- e.backtrace.each {|m| Kameleon.logger.debug m }
46
- exit 16
47
- rescue Exception => e
48
- if ENV["KAMELEON_LOG"] != "debug"
49
- $stderr << "Unfortunately, a fatal error has occurred : "\
50
- "#{e.message}.\nUse --debug option for more details\n"
51
- else
52
- Kameleon.logger.debug "Error : #{e}"
53
- e.backtrace.each {|m| puts "==> #{m}" }
54
- end
55
- exit 666
56
- end
57
26
  end
@@ -1,7 +1,6 @@
1
1
  require 'childprocess'
2
2
  require 'singleton'
3
3
  require 'socket'
4
-
5
4
  module Kameleon
6
5
  #This ruby class will control the execution of Polipo web proxy
7
6
  class Persistent_cache
@@ -12,11 +11,15 @@ module Kameleon
12
11
  attr_reader :polipo_port
13
12
  attr_writer :activated
14
13
  attr_reader :cwd
15
- attr_reader :polipo_path
14
+ attr_writer :polipo_path
16
15
  attr_reader :name
16
+ attr_writer :cache_path
17
+ attr_accessor :mode
18
+ attr_accessor :name
19
+ attr_accessor :recipe_files # have to check those.
20
+ attr_accessor :recipe_path
17
21
 
18
22
  def initialize()
19
- @logger = Log4r::Logger.new("kameleon::[kameleon]")
20
23
  ## we must configure Polipo to be execute for the in and out context
21
24
  ## we have to start polipo in the out context for debootstrap step
22
25
 
@@ -28,18 +31,28 @@ module Kameleon
28
31
  @polipo_port = find_unused_port
29
32
 
30
33
  @polipo_cmd_options = {:diskCacheRoot => "",
31
- :idleTime => "5",
32
- :chunkHighMark => "425165824",
33
- :proxyPort => @polipo_port,
34
- :relaxTransparency =>"true"
34
+ :idleTime => "1",
35
+ :chunkHighMark => "425165824",
36
+ :proxyPort => @polipo_port,
37
+ :relaxTransparency =>"true",
38
+ :daemonise => false,
39
+ :logFile => File.join(Kameleon.env.build_path, 'polipo.log')
35
40
  }
36
41
 
37
42
  @activated = false
38
43
 
39
- @cache_dir = ""
44
+ @mode = nil #It could be build or from
45
+ @cache_dir = Kameleon.env.cache_path
40
46
  @polipo_path = nil
41
47
  @cwd = ""
42
-
48
+ #structure {:cmd => "cmd", :stdout_filename => "file_name"}
49
+ @cmd_cached = []
50
+ @cache_path = ""
51
+ @current_cmd_id = nil
52
+ @current_step_dir = nil
53
+ @recipe_file = nil
54
+ @steps_files = []
55
+ @cached_recipe_dir = nil
43
56
  end
44
57
 
45
58
  def find_unused_port
@@ -49,7 +62,7 @@ module Kameleon
49
62
  ports.each do |p|
50
63
  begin
51
64
  port = p
52
- tmp = TCPServer.new('localhost',port)
65
+ tmp = TCPServer.new('localhost', port)
53
66
  rescue
54
67
  port =0
55
68
  end
@@ -61,12 +74,11 @@ module Kameleon
61
74
 
62
75
  def check_polipo_binary
63
76
 
64
-
65
77
  @polipo_path ||= which("polipo")
66
78
 
67
79
  if @polipo_path.nil? then
68
- @logger.error("Polipo binary not found, make sure it is in your current PATH")
69
- @logger.error("or use the option --proxy-path")
80
+ Kameleon.ui.error("Polipo binary not found, make sure it is in your current PATH")
81
+ Kameleon.ui.error("or use the option --proxy-path")
70
82
  raise BuildError, "Failed to use persistent cache"
71
83
  end
72
84
  end
@@ -75,54 +87,131 @@ module Kameleon
75
87
  @activated
76
88
  end
77
89
 
78
-
79
90
  def cwd=(dir)
80
91
  @cwd = dir
81
- @cache_dir = @cwd + "/cache/"
82
92
  end
83
93
 
84
94
  def create_cache_directory(step_name)
85
- @logger.notice("Creating cache directory #{step_name} for Polipo")
86
- directory_name = @cache_dir + "/#{step_name}"
95
+ Kameleon.ui.debug("Creating cache directory #{step_name} for Polipo")
96
+ directory_name = File.join(@cache_dir,"#{step_name}")
87
97
  FileUtils.mkdir_p directory_name
88
98
  directory_name
89
99
  end
90
100
 
91
101
  def start_web_proxy_in(directory)
102
+
103
+ ## setting current step dir
104
+ @current_step_dir = directory
92
105
  ## This function assumes that the cache directory has already been created by the engine
93
106
  ## Stopping first the previous proxy
94
107
  ## have to check if polipo is running
95
- @logger.debug("Starting web proxy Polipo in directory #{directory} using port: #{@polipo_port}")
96
- @polipo_process.stop unless @polipo_process.nil?
97
- command = ["#{@polipo_path}/polipo"]
108
+ Kameleon.ui.debug("Starting web proxy Polipo in directory #{directory} using port: #{@polipo_port}")
109
+ @polipo_process.stop(0) unless @polipo_process.nil?
110
+ command = ["#{@polipo_path}/polipo", "-c", "/dev/null"]
98
111
  @polipo_cmd_options[:diskCacheRoot] = directory
99
112
  @polipo_cmd_options.each{ |v,k| command.push("#{v}=#{k}") }
100
113
  ChildProcess.posix_spawn = true
114
+ Kameleon.ui.debug("Starting process '#{command}'")
101
115
  @polipo_process = ChildProcess.build(*command)
102
- @polipo_process.io.stdout = Tempfile.new("polipo_output")
103
116
  @polipo_process.start
117
+ return true
104
118
  end
105
119
 
106
120
 
107
121
  def stop_web_proxy
108
122
  @polipo_process.stop
109
- @logger.notice("Stopping web proxy polipo")
123
+ Kameleon.ui.info("Stopping web proxy polipo")
110
124
  end
111
125
 
112
126
  def pack()
113
- @logger.notice("Packing up the generated cache in #{@cwd}")
114
- execute("tar","-cf #{@name}-cache.tar cache/",@cwd)
115
- # The cache directory cannot be deleted due to the checkpoints
127
+ Kameleon.ui.info("Packing up the generated cache in #{@cwd}/#{@name}-cache.tar")
128
+ execute("tar","-cf #{@name}-cache.tar -C cache/ .",@cwd)
116
129
  end
117
130
 
118
131
  def unpack(cache_path)
119
- @logger.notice("Unpacking persistent cache: #{cache_path}")
120
- execute("tar","-xf #{cache_path} -C #{@cwd}")
132
+ Kameleon.ui.info("Unpacking persistent cache: #{cache_path}")
133
+ FileUtils.mkdir_p @cache_dir
134
+ execute("tar","-xf #{cache_path} -C #{@cache_dir}")
135
+ end
136
+
137
+
138
+ # This function caches the command with its respective stdout
139
+ # a command id is associate to a file
140
+ def cache_cmd_id(cmd_identifier)
141
+ @current_cmd_id = cmd_identifier
142
+ return true
143
+ end
144
+
145
+ def cache_cmd(cmd,file_path)
146
+ Kameleon.ui.info("Caching file")
147
+ Kameleon.ui.debug("command: cp #{file_path} #{@cwd}/cache/files/")
148
+ FileUtils.mkdir_p @current_step_dir + "/data/"
149
+ FileUtils.cp file_path, @current_step_dir + "/data/"
150
+ @cmd_cached.push({:cmd_id => @current_cmd_id,
151
+ :cmd => cmd ,
152
+ :stdout_filename => File.basename(file_path)})
153
+ end
154
+
155
+ def get_cache_cmd(cmd)
156
+ return false if @mode == :build
157
+ cache_line = @cmd_cached.select{ |reg|
158
+ (reg[:cmd_id] == @current_cmd_id && reg[:cmd] == cmd) }.first
159
+
160
+ return File.new("#{@current_step_dir}/data/#{cache_line[:stdout_filename]}","r")
161
+ end
162
+
163
+ def stop()
164
+ @polipo_process.stop
165
+ Kameleon.ui.info("Stopping web proxy polipo")
166
+ Kameleon.ui.info("Finishing persistent cache with last files")
167
+
168
+ if @mode == :build then
169
+ File.open("#{@cache_dir}/cache_cmd_index",'w+') do |f|
170
+ f.puts(@cmd_cached.to_yaml)
171
+ end
172
+
173
+ unless @recipe_files.empty?
174
+ recipe_dir = Pathname.new(common_prefix(@recipe_files))
175
+ all_files = @recipe_files.push(@recipe_path)
176
+ Kameleon::Utils.copy_files(recipe_dir, @cache_dir, all_files)
177
+ end
178
+ ## Saving metadata information
179
+ Kameleon.ui.info("Caching recipe")
180
+ File.open("#{@cached_recipe_dir}/header",'w+') do |f|
181
+ f.puts({:recipe_path => @recipe_path.to_s}.to_yaml)
182
+ end
183
+ pack
184
+ end
185
+ end
186
+
187
+ def start()
188
+ check_polipo_binary
189
+ if @mode == :from then
190
+ begin
191
+ unpack(@cache_path)
192
+ rescue
193
+ raise BuildError, "Failed to untar the persistent cache file"
194
+ end
195
+ ## We have to load the file
196
+ @cmd_cached = YAML.load(File.read("#{@cache_dir}/cache_cmd_index"))
197
+ end
198
+ @activated = true
199
+ @cached_recipe_dir = @cache_dir
200
+ FileUtils.mkdir_p @cached_recipe_dir
201
+ end
202
+
203
+ def get_recipe()
204
+ cached_recipe=Dir.mktmpdir("cache")
205
+ puts "cache path : #{@cache_path}"
206
+ execute("tar","-xf #{@cache_path} -C #{cached_recipe} .")
207
+ Kameleon.ui.info("Getting cached recipe")
208
+ recipe_header = YAML::load(File.read("#{cached_recipe}/header"))
209
+ recipe_file = recipe_header[:recipe_path]
210
+ return recipe_file
121
211
  end
122
212
 
123
213
  def execute(cmd,args,dir=nil)
124
214
  command = [cmd ] + args.split(" ")
125
- # @logger.notice(" command generated: #{command}")
126
215
  process = ChildProcess.build(*command)
127
216
  process.cwd = dir unless dir.nil?
128
217
  process.start
@@ -137,8 +226,17 @@ module Kameleon
137
226
  return nil
138
227
  end
139
228
 
229
+ def common_prefix(paths)
230
+ return '' if paths.empty?
231
+ return paths.first.split('/').slice(0...-1).join('/') if paths.length <= 1
232
+ arr = paths.sort
233
+ first = arr.first.to_s.split('/')
234
+ last = arr.last.to_s.split('/')
235
+ i = 0
236
+ i += 1 while first[i] == last[i] && i <= first.length
237
+ first.slice(0, i).join('/')
238
+ end
140
239
 
141
240
  end
142
241
 
143
242
  end
144
-
@@ -13,9 +13,11 @@ module Kameleon
13
13
  attr_accessor :checkpoint
14
14
  attr_accessor :checkpoint_path
15
15
  attr_accessor :metainfo
16
+ attr_accessor :files
17
+ attr_accessor :base_recipes_files
18
+
16
19
 
17
20
  def initialize(path, kwargs = {})
18
- @logger = Log4r::Logger.new("kameleon::[kameleon]")
19
21
  @path = Pathname.new(File.expand_path(path))
20
22
  @name = (@path.basename ".yaml").to_s
21
23
  @recipe_content = File.open(@path, 'r') { |f| f.read }
@@ -24,27 +26,24 @@ module Kameleon
24
26
  "setup" => Section.new("setup"),
25
27
  "export" => Section.new("export"),
26
28
  }
27
- kameleon_id = SecureRandom.uuid
28
29
  @global = {
29
30
  "kameleon_recipe_name" => @name,
30
31
  "kameleon_recipe_dir" => File.dirname(@path),
31
- "kameleon_uuid" => kameleon_id,
32
- "kameleon_short_uuid" => kameleon_id.split("-").last,
33
32
  "kameleon_cwd" => File.join(Kameleon.env.build_path, @name),
34
- "in_context" => {"cmd"=> "/bin/bash"},
35
- "out_context" => {"cmd"=> "/bin/bash"}
33
+ "in_context" => {"cmd" => "/bin/bash", "proxy_cache" => "localhost"},
34
+ "out_context" => {"cmd" => "/bin/bash", "proxy_cache" => "localhost"}
36
35
  }
37
36
  @aliases = {}
38
37
  @checkpoint = nil
39
38
  @files = []
40
- @logger.debug("Initialize new recipe (#{path})")
39
+ Kameleon.ui.debug("Initialize new recipe (#{path})")
41
40
  @base_recipes_files = [@path]
42
- load! kwargs
41
+ load! :strict => false
43
42
  end
44
43
 
45
44
  def load!(kwargs = {})
46
45
  # Find recipe path
47
- @logger.debug("Loading #{@path}")
46
+ Kameleon.ui.debug("Loading #{@path}")
48
47
  fail RecipeError, "Could not find this following recipe : #{@path}" \
49
48
  unless File.file? @path
50
49
  yaml_recipe = YAML.load File.open @path
@@ -59,22 +58,27 @@ module Kameleon
59
58
  @global.merge!(yaml_recipe.fetch("global", {}))
60
59
  # Resolve dynamically-defined variables !!
61
60
  resolved_global = Utils.resolve_vars(@global.to_yaml, @path, @global, kwargs)
62
- @global.merge! YAML.load(resolved_global)
61
+ resolved_global = @global.merge YAML.load(resolved_global)
63
62
  # Loads aliases
64
63
  load_aliases(yaml_recipe)
65
64
  # Loads checkpoint configuration
66
65
  load_checkpoint_config(yaml_recipe)
67
66
 
68
67
  #Find and load steps
69
- steps_dir = File.join(File.dirname(@path), 'steps')
70
- @global['include_steps'] ||= []
71
- @global['include_steps'] = [global['include_steps']].push ''
72
- @global['include_steps'].flatten!
73
- @global['include_steps'].compact!
68
+ steps_dirs = [
69
+ File.join(File.dirname(@path), 'steps'),
70
+ File.expand_path(File.join(File.dirname(@path), '..', 'steps')),
71
+ ]
72
+ resolved_global['include_steps'] ||= []
73
+ resolved_global['include_steps'].push ''
74
+ resolved_global['include_steps'].flatten!
75
+ resolved_global['include_steps'].compact!
74
76
  @sections.values.each do |section|
75
- dir_to_search = @global['include_steps'].map do |path|
76
- [File.join(steps_dir, section.name, path),
77
- File.join(steps_dir, path)]
77
+ dir_to_search = resolved_global['include_steps'].map do |path|
78
+ steps_dirs.map do |steps_dir|
79
+ [File.join(steps_dir, section.name, path),
80
+ File.join(steps_dir, path)]
81
+ end
78
82
  end
79
83
  dir_to_search.flatten!
80
84
  if yaml_recipe.key? section.name
@@ -104,7 +108,7 @@ module Kameleon
104
108
  end
105
109
  end
106
110
  if embedded_step
107
- @logger.debug("Loading embedded macrostep #{name}")
111
+ Kameleon.ui.debug("Loading embedded macrostep #{name}")
108
112
  macrostep = load_macrostep(nil, name, args)
109
113
  section.macrosteps.push(macrostep)
110
114
  next
@@ -114,16 +118,16 @@ module Kameleon
114
118
  dir_to_search.each do |dir|
115
119
  macrostep_path = Pathname.new(File.join(dir, name + '.yaml'))
116
120
  if File.file?(macrostep_path)
117
- @logger.debug("Loading macrostep #{macrostep_path}")
121
+ Kameleon.ui.debug("Loading macrostep #{macrostep_path}")
118
122
  macrostep = load_macrostep(macrostep_path, name, args)
119
123
  section.macrosteps.push(macrostep)
120
124
  @files.push(macrostep_path)
121
- @logger.debug("Macrostep '#{name}' found in this path: " \
125
+ Kameleon.ui.debug("Macrostep '#{name}' found in this path: " \
122
126
  "#{macrostep_path}")
123
127
  loaded = true
124
128
  break
125
129
  else
126
- @logger.debug("Macrostep '#{name}' not found in this path: " \
130
+ Kameleon.ui.debug("Macrostep '#{name}' not found in this path: " \
127
131
  "#{macrostep_path}")
128
132
  end
129
133
  end
@@ -131,7 +135,7 @@ module Kameleon
131
135
  end
132
136
  end
133
137
  end
134
- @logger.debug("Loading recipe metadata")
138
+ Kameleon.ui.debug("Loading recipe metadata")
135
139
  @metainfo = {
136
140
  "description" => Utils.extract_meta_var("description", @recipe_content)
137
141
  }
@@ -194,12 +198,13 @@ module Kameleon
194
198
  elsif aliases.kind_of? String
195
199
  dir_search = [
196
200
  File.join(File.dirname(@path), "steps", "aliases"),
201
+ File.join(File.dirname(@path), "..", "steps", "aliases"),
197
202
  File.join(File.dirname(@path), "aliases")
198
203
  ]
199
204
  dir_search.each do |dir_path|
200
205
  path = Pathname.new(File.join(dir_path, aliases))
201
206
  if File.file?(path)
202
- @logger.debug("Loading aliases #{path}")
207
+ Kameleon.ui.debug("Loading aliases #{path}")
203
208
  @aliases = YAML.load_file(path)
204
209
  @files.push(path)
205
210
  return path
@@ -219,20 +224,25 @@ module Kameleon
219
224
  elsif checkpoint.kind_of? String
220
225
  dir_search = [
221
226
  File.join(File.dirname(@path), "steps", "checkpoints"),
227
+ File.join(File.dirname(@path), "..", "steps", "checkpoints"),
222
228
  File.join(File.dirname(@path), "checkpoints")
223
229
  ]
224
230
  dir_search.each do |dir_path|
225
231
  path = Pathname.new(File.join(dir_path, checkpoint))
226
232
  if File.file?(path)
227
- @logger.debug("Loading checkpoint configuration #{path}")
233
+ Kameleon.ui.debug("Loading checkpoint configuration #{path}")
228
234
  @checkpoint = YAML.load_file(path)
229
235
  @checkpoint["path"] = path.to_s
230
236
  @files.push(path)
231
- return path
232
237
  end
233
238
  end
234
239
  fail RecipeError, "Checkpoint configuraiton file '#{path}' " \
235
- "does not exists"
240
+ "does not exists" if @checkpoint.nil?
241
+ end
242
+ (@checkpoint.keys - ["path"]).each do |key|
243
+ @checkpoint[key].map! do |cmd|
244
+ Kameleon::Command.new(cmd, "checkpoint")
245
+ end
236
246
  end
237
247
  end
238
248
  end
@@ -295,15 +305,21 @@ module Kameleon
295
305
  end
296
306
 
297
307
  def find_microstep(microstep_name, loaded_microsteps)
298
- @logger.debug("Looking for microstep #{microstep_name}")
308
+ Kameleon.ui.debug("Looking for microstep #{microstep_name}")
299
309
  loaded_microsteps.each do |microstep|
300
310
  if microstep_name.eql? microstep.name
301
311
  return microstep
302
312
  end
303
313
  end
314
+ return nil
304
315
  end
305
316
 
306
317
  def resolve!
318
+ unless @global.keys.include? "kameleon_uuid"
319
+ kameleon_id = SecureRandom.uuid
320
+ @global["kameleon_uuid"] = kameleon_id
321
+ @global["kameleon_short_uuid"] = kameleon_id.split("-").last
322
+ end
307
323
  # Resolve dynamically-defined variables !!
308
324
  resolved_global = Utils.resolve_vars(@global.to_yaml, @path, @global)
309
325
  @global.merge! YAML.load(resolved_global)
@@ -311,7 +327,7 @@ module Kameleon
311
327
  consistency_check
312
328
  resolve_checkpoint unless @checkpoint.nil?
313
329
 
314
- @logger.notice("Resolving variables")
330
+ Kameleon.ui.info("Resolving variables")
315
331
  @sections.values.each do |section|
316
332
  section.macrosteps.each do |macrostep|
317
333
  macrostep.resolve_variables!(@global)
@@ -321,7 +337,7 @@ module Kameleon
321
337
  @sections.values.each do |section|
322
338
  section.macrosteps.each do |macrostep|
323
339
  # First pass : resolve aliases
324
- @logger.debug("Resolving aliases for macrostep '#{macrostep.name}'")
340
+ Kameleon.ui.debug("Resolving aliases for macrostep '#{macrostep.name}'")
325
341
  macrostep.microsteps.each do |microstep|
326
342
  microstep.commands.map! do |cmd|
327
343
  # resolve alias
@@ -329,7 +345,7 @@ module Kameleon
329
345
  end
330
346
  end
331
347
  # flatten for multiple-command alias + variables
332
- @logger.debug("Resolving check statements for macrostep '#{macrostep.name}'")
348
+ Kameleon.ui.debug("Resolving check statements for macrostep '#{macrostep.name}'")
333
349
  macrostep.microsteps.each { |microstep| microstep.commands.flatten! }
334
350
  # Second pass : resolve variables + clean/init hooks
335
351
  macrostep.microsteps.each do |microstep|
@@ -337,7 +353,7 @@ module Kameleon
337
353
  resolve_hooks(cmd, macrostep, microstep)
338
354
  end
339
355
  end
340
- @logger.debug("Compacting macrostep '#{macrostep.name}'")
356
+ Kameleon.ui.debug("Compacting macrostep '#{macrostep.name}'")
341
357
  # remove empty steps
342
358
  macrostep.microsteps.map! do |microstep|
343
359
  microstep.commands.compact!
@@ -345,7 +361,7 @@ module Kameleon
345
361
  end
346
362
  # remove nil values
347
363
  macrostep.microsteps.compact!
348
- @logger.debug("Resolving commands for macrostep '#{macrostep.name}'")
364
+ Kameleon.ui.debug("Resolving commands for macrostep '#{macrostep.name}'")
349
365
  macrostep.microsteps.each do |microstep|
350
366
  microstep.resolve!
351
367
  end
@@ -365,7 +381,7 @@ module Kameleon
365
381
  end
366
382
  end
367
383
  end
368
- @logger.notice("Starting recipe consistency check")
384
+ Kameleon.ui.info("Starting recipe consistency check")
369
385
  # check context args
370
386
  required_args = %w(cmd)
371
387
  missings = []
@@ -385,10 +401,12 @@ module Kameleon
385
401
  end
386
402
 
387
403
  def resolve_checkpoint()
388
- %w(create apply list clear).each do |key|
389
- @checkpoint[key] = Utils.resolve_vars(@checkpoint[key],
404
+ (@checkpoint.keys - ["path"]).each do |key|
405
+ @checkpoint[key].each do |cmd|
406
+ cmd.string_cmd = Utils.resolve_vars(cmd.string_cmd,
390
407
  @checkpoint["path"],
391
408
  @global)
409
+ end
392
410
  end
393
411
  end
394
412
 
@@ -490,17 +508,21 @@ module Kameleon
490
508
  end
491
509
 
492
510
  def calculate_step_identifiers
493
- @logger.notice("Calculating microstep identifiers")
511
+ Kameleon.ui.debug("Calculating microstep identifiers")
494
512
  base_salt = ""
495
513
  order = 0
496
514
  @sections.values.each do |section|
497
515
  section.sequence do |macrostep|
498
516
  macrostep.sequence do |microstep|
499
- base_salt = microstep.calculate_identifier base_salt
517
+ if ["redo", "skip"].include? microstep.on_checkpoint
518
+ microstep.calculate_identifier ""
519
+ else
520
+ base_salt = microstep.calculate_identifier base_salt
521
+ end
500
522
  slug = "#{section.name}/#{macrostep.name}/#{microstep.name}"
501
523
  microstep.slug = slug
502
524
  microstep.order = (order += 1)
503
- @logger.debug(" #{microstep.slug}: #{microstep.identifier}")
525
+ Kameleon.ui.debug(" #{microstep.slug}: #{microstep.identifier}")
504
526
  end
505
527
  end
506
528
  end
@@ -531,73 +553,8 @@ module Kameleon
531
553
  end
532
554
 
533
555
  class RecipeTemplate < Recipe
534
-
535
- def get_answer(msg)
536
- while true
537
- @logger.progress_notice msg
538
- answer = $stdin.gets.downcase
539
- raise AbortError, "Execution aborted..." if answer.nil?
540
- answer.chomp!
541
- if ["y", "n" , "", "a"].include?(answer)
542
- if ["y", ""].include? answer
543
- return true
544
- elsif answer.eql? "a"
545
- raise AbortError, "Aborted..."
546
- end
547
- return false
548
- end
549
- end
550
- end
551
-
552
- def safe_copy_file(src, dst, force)
553
- if File.exists? dst
554
- diff = Diffy::Diff.new(dst.to_s, src.to_s, :source => "files").to_s
555
- unless diff.chomp.empty?
556
- @logger.notice("conflict #{dst}")
557
- @logger.notice("Differences between the old and the new :")
558
- puts Diffy::Diff.new(dst.to_s, src.to_s,
559
- :source => "files",
560
- :context => 1,
561
- :include_diff_info => true).to_s
562
- msg = "Overwrite #{dst}? [Y]es/[n]o/[a]bort : "
563
- if force || get_answer(msg)
564
- FileUtils.copy_file(src, dst)
565
- end
566
- else
567
- @logger.notice("identical #{dst}")
568
- end
569
- else
570
- unless File.dirname(dst).eql? "/"
571
- FileUtils.mkdir_p File.dirname(dst)
572
- end
573
- @logger.notice("create #{dst}")
574
- FileUtils.copy_file(src, dst)
575
- end
576
- end
577
-
578
- def copy_extended_recipe(recipe_name, force)
579
- Dir::mktmpdir do |tmp_dir|
580
- recipe_path = File.join(tmp_dir, recipe_name + '.yaml')
581
- ## copying recipe
582
- File.open(recipe_path, 'w+') do |file|
583
- extend_erb_tpl = File.join(Kameleon.env.templates_path, "extend.erb")
584
- tpl = ERB.new(File.open(extend_erb_tpl, 'rb') { |f| f.read })
585
- result = tpl.result(binding)
586
- file.write(result)
587
- end
588
- recipe_dst = File.join(Kameleon.env.workspace, recipe_name + '.yaml')
589
- safe_copy_file(recipe_path, Pathname.new(recipe_dst), force)
590
- end
591
- end
592
-
593
- def copy_template(force)
594
- ## copying steps
595
- files2copy = @base_recipes_files + @files
596
- files2copy.each do |path|
597
- relative_path = path.relative_path_from(Kameleon.env.templates_path)
598
- dst = File.join(Kameleon.env.workspace, relative_path)
599
- safe_copy_file(path, dst, force)
600
- end
556
+ def relative_path()
557
+ @path.relative_path_from(Kameleon.env.templates_path)
601
558
  end
602
559
  end
603
560
  end