kameleon-builder 2.1.3 → 2.2.0
Sign up to get free protection for your applications and to get access to all the features.
- data/.editorconfig +0 -0
- data/.env +2 -6
- data/CHANGELOG.rst +74 -1
- data/Gemfile +20 -2
- data/README.rst +35 -25
- data/Thorfile +29 -0
- data/contrib/kameleon_bashrc.sh +61 -0
- data/contrib/polipo_env.sh +3 -2
- data/kameleon-builder.gemspec +3 -10
- data/lib/kameleon.rb +10 -14
- data/lib/kameleon/cli.rb +92 -105
- data/lib/kameleon/context.rb +76 -43
- data/lib/kameleon/engine.rb +147 -103
- data/lib/kameleon/environment.rb +10 -5
- data/lib/kameleon/error.rb +1 -32
- data/lib/kameleon/persistent_cache.rb +127 -29
- data/lib/kameleon/recipe.rb +63 -106
- data/lib/kameleon/shell.rb +32 -17
- data/lib/kameleon/step.rb +18 -1
- data/lib/kameleon/ui.rb +141 -0
- data/lib/kameleon/utils.rb +9 -0
- data/templates/{debian7-chroot.yaml → chroot/debian7-amd64.yaml} +29 -21
- data/templates/{docker-debian7.yaml → docker/debian7.yaml} +5 -5
- data/templates/extend.erb +2 -2
- data/templates/{debian7-g5k.yaml → grid5000/debian7.yaml} +1 -1
- data/templates/{archlinux-desktop.yaml → qemu/archlinux-desktop-i686.yaml} +2 -2
- data/templates/qemu/archlinux-desktop-x86_64.yaml +25 -0
- data/templates/qemu/archlinux-i686.yaml +25 -0
- data/templates/{archlinux.yaml → qemu/archlinux-x86_64.yaml} +37 -34
- data/templates/qemu/centos6.5-x86_64.yaml +113 -0
- data/templates/qemu/centos7-x86_64.yaml +119 -0
- data/templates/{debian7.yaml → qemu/debian7-amd64.yaml} +45 -40
- data/templates/{debian7-desktop.yaml → qemu/debian7-desktop-amd64.yaml} +2 -3
- data/templates/{debian7-kameleon.yaml → qemu/debian7-kameleon-amd64.yaml} +3 -6
- data/templates/{debian7-oar-dev.yaml → qemu/debian7-oar-amd64.yaml} +2 -2
- data/templates/qemu/debian8-amd64.yaml +25 -0
- data/templates/{fedora20-desktop.yaml → qemu/fedora20-desktop-x86_64.yaml} +2 -2
- data/templates/qemu/fedora20-x86_64.yaml +116 -0
- data/templates/{ubuntu-12.04.yaml → qemu/ubuntu-12.04-amd64.yaml} +42 -38
- data/templates/{ubuntu-12.04-desktop.yaml → qemu/ubuntu-12.04-desktop-amd64.yaml} +3 -3
- data/templates/{ubuntu-14.04.yaml → qemu/ubuntu-14.04-amd64.yaml} +2 -2
- data/templates/{ubuntu-14.04-desktop.yaml → qemu/ubuntu-14.04-desktop-amd64.yaml} +2 -2
- data/templates/steps/aliases/defaults.yaml +19 -13
- data/templates/steps/bootstrap/archlinux/arch_bootstrap.yaml +11 -19
- data/templates/steps/bootstrap/debian/debootstrap.yaml +10 -11
- data/templates/steps/bootstrap/debian/debootstrap_arm.yaml +11 -19
- data/templates/steps/bootstrap/fedora/init_pxeboot.yaml +13 -0
- data/templates/steps/bootstrap/fedora/yum_bootstrap.yaml +45 -0
- data/templates/steps/bootstrap/initialize_disk.yaml +16 -0
- data/templates/steps/bootstrap/initialize_disk_chroot.yaml +10 -14
- data/templates/steps/bootstrap/install_requirements.yaml +3 -0
- data/templates/steps/bootstrap/prepare_docker.yaml +7 -7
- data/templates/steps/bootstrap/prepare_qemu.yaml +20 -26
- data/templates/steps/bootstrap/prepare_virtualbox.yaml +63 -0
- data/templates/steps/bootstrap/start_chroot.yaml +2 -13
- data/templates/steps/bootstrap/start_qemu.yaml +50 -53
- data/templates/steps/bootstrap/start_virtualbox.yaml +37 -0
- data/templates/steps/bootstrap/switch_context_qemu.yaml +64 -0
- data/templates/steps/bootstrap/switch_context_virtualbox.yaml +66 -0
- data/templates/steps/breakpoint.yaml +2 -1
- data/templates/steps/checkpoints/docker.yaml +14 -11
- data/templates/steps/checkpoints/qcow2.yaml +26 -24
- data/templates/steps/checkpoints/qemu.yaml +30 -36
- data/templates/steps/checkpoints/virtualbox.yaml +21 -0
- data/templates/steps/disable_checkpoint.yaml +3 -0
- data/templates/steps/enable_checkpoint.yaml +5 -0
- data/templates/steps/export/qemu_save_appliance.yaml +52 -0
- data/templates/steps/export/virtualbox_save_appliance.yaml +67 -0
- data/templates/steps/setup/archlinux/install_bootloader.yaml +23 -0
- data/templates/steps/setup/centos/6.5/configure_network.yaml +31 -0
- data/templates/steps/setup/centos/6.5/configure_system.yaml +27 -0
- data/templates/steps/setup/debian/configure_apt.yaml +1 -6
- data/templates/steps/setup/debian/configure_kernel.yaml +0 -5
- data/templates/steps/setup/debian/install_bootloader.yaml +36 -0
- data/templates/steps/setup/debian/setup_vagrant_box.yaml +48 -18
- data/templates/steps/setup/debian/upgrade_system.yaml +2 -7
- data/templates/steps/setup/fedora/configure_kernel.yaml +9 -0
- data/templates/steps/setup/fedora/configure_keyboard.yaml +12 -0
- data/templates/steps/setup/fedora/configure_network.yaml +9 -1
- data/templates/steps/setup/fedora/configure_system.yaml +20 -45
- data/templates/steps/setup/fedora/install_bootloader.yaml +58 -0
- data/templates/steps/setup/fedora/minimal_install.yaml +3 -0
- data/templates/steps/setup/fedora/update_system.yaml +13 -5
- data/templates/steps/setup/ubuntu/configure_apt.yaml +4 -9
- data/templates/vagrant/debian7-amd64.yaml +130 -0
- data/templates/virtualbox/archlinux-desktop-i686.yaml +25 -0
- data/templates/virtualbox/archlinux-desktop-x86_64.yaml +28 -0
- data/templates/virtualbox/archlinux-i686.yaml +28 -0
- data/templates/virtualbox/archlinux-x86_64.yaml +109 -0
- data/templates/virtualbox/centos6.5-i386.yaml +39 -0
- data/templates/virtualbox/centos6.5-x86_64.yaml +111 -0
- data/templates/virtualbox/centos7-x86_64.yaml +116 -0
- data/{docs/source/debian7.yaml → templates/virtualbox/debian7-amd64.yaml} +53 -52
- data/templates/virtualbox/debian7-desktop-amd64.yaml +25 -0
- data/templates/{fedora-rawhide.yaml → virtualbox/debian7-i386.yaml} +12 -11
- data/templates/virtualbox/debian7-kameleon-amd64.yaml +38 -0
- data/templates/virtualbox/debian7-oar-amd64.yaml +51 -0
- data/templates/{debian-testing.yaml → virtualbox/debian8-amd64.yaml} +3 -3
- data/templates/virtualbox/debian8-i386.yaml +31 -0
- data/templates/virtualbox/fedora20-x86_64.yaml +116 -0
- data/templates/virtualbox/ubuntu-12.04-amd64.yaml +128 -0
- data/templates/virtualbox/ubuntu-12.04-desktop-amd64.yaml +25 -0
- data/templates/virtualbox/ubuntu-14.04-amd64.yaml +25 -0
- data/templates/virtualbox/ubuntu-14.04-desktop-amd64.yaml +27 -0
- data/version.txt +1 -1
- metadata +67 -212
- data/Rakefile +0 -24
- data/docs/.gitignore +0 -1
- data/docs/Makefile +0 -181
- data/docs/README.md +0 -17
- data/docs/make.bat +0 -242
- data/docs/source/_static/.gitignore +0 -0
- data/docs/source/_static/centos.png +0 -0
- data/docs/source/_static/debian.png +0 -0
- data/docs/source/_static/kameleon-logo.png +0 -0
- data/docs/source/_static/kameleon-logo.xcf +0 -0
- data/docs/source/_static/kameleon-long.png +0 -0
- data/docs/source/_static/ubuntu.png +0 -0
- data/docs/source/_themes/sphinx_rtd_theme/__init__.py +0 -17
- data/docs/source/_themes/sphinx_rtd_theme/breadcrumbs.html +0 -19
- data/docs/source/_themes/sphinx_rtd_theme/footer.html +0 -32
- data/docs/source/_themes/sphinx_rtd_theme/layout.html +0 -160
- data/docs/source/_themes/sphinx_rtd_theme/layout_old.html +0 -205
- data/docs/source/_themes/sphinx_rtd_theme/search.html +0 -50
- data/docs/source/_themes/sphinx_rtd_theme/searchbox.html +0 -7
- data/docs/source/_themes/sphinx_rtd_theme/static/css/badge_only.css +0 -1
- data/docs/source/_themes/sphinx_rtd_theme/static/css/theme.css +0 -4
- data/docs/source/_themes/sphinx_rtd_theme/static/fonts/FontAwesome.otf +0 -0
- data/docs/source/_themes/sphinx_rtd_theme/static/fonts/fontawesome-webfont.eot +0 -0
- data/docs/source/_themes/sphinx_rtd_theme/static/fonts/fontawesome-webfont.svg +0 -414
- data/docs/source/_themes/sphinx_rtd_theme/static/fonts/fontawesome-webfont.ttf +0 -0
- data/docs/source/_themes/sphinx_rtd_theme/static/fonts/fontawesome-webfont.woff +0 -0
- data/docs/source/_themes/sphinx_rtd_theme/static/js/theme.js +0 -47
- data/docs/source/_themes/sphinx_rtd_theme/theme.conf +0 -8
- data/docs/source/_themes/sphinx_rtd_theme/versions.html +0 -37
- data/docs/source/aliases.rst +0 -31
- data/docs/source/atlas_debian_g5k.yaml +0 -36
- data/docs/source/checkpoint.rst +0 -30
- data/docs/source/commands.rst +0 -63
- data/docs/source/conf.py +0 -262
- data/docs/source/context.rst +0 -47
- data/docs/source/debian_customized.yaml +0 -28
- data/docs/source/debian_customized_g5k.yaml +0 -21
- data/docs/source/faq.rst +0 -43
- data/docs/source/getting_started.rst +0 -260
- data/docs/source/grid5000_tutorial.rst +0 -525
- data/docs/source/index.rst +0 -53
- data/docs/source/install_atlas.yaml +0 -25
- data/docs/source/install_hpl.yaml +0 -24
- data/docs/source/installation.rst +0 -51
- data/docs/source/persistent_cache.rst +0 -34
- data/docs/source/recipe.rst +0 -177
- data/docs/source/tau_install.yaml +0 -19
- data/docs/source/tau_install_g5k.yaml +0 -25
- data/docs/source/use_cases.rst +0 -93
- data/docs/source/workspace.rst +0 -13
- data/lib/kameleon/logger.rb +0 -56
- data/omnibus/.gitignore +0 -11
- data/omnibus/.kitchen.yml +0 -25
- data/omnibus/Berksfile +0 -9
- data/omnibus/Berksfile.lock +0 -25
- data/omnibus/Gemfile +0 -12
- data/omnibus/README.md +0 -94
- data/omnibus/config/projects/kameleon.rb +0 -23
- data/omnibus/config/software/kameleon.rb +0 -24
- data/omnibus/config/software/polipo.rb +0 -30
- data/omnibus/config/software/ruby.rb +0 -158
- data/omnibus/files/mac_dmg/Resources/background.png +0 -0
- data/omnibus/files/mac_dmg/Resources/icon.png +0 -0
- data/omnibus/files/mac_pkg/Resources/background.png +0 -0
- data/omnibus/files/mac_pkg/Resources/license.html +0 -1
- data/omnibus/files/mac_pkg/Resources/welcome.html +0 -9
- data/omnibus/omnibus.rb +0 -27
- data/omnibus/package-scripts/kameleon/makeselfinst +0 -27
- data/omnibus/package-scripts/kameleon/postrm +0 -9
- data/templates/fedora20.yaml +0 -105
- data/templates/steps/bootstrap/archlinux/install_bootloader.yaml +0 -46
- data/templates/steps/bootstrap/archlinux/populate_disk.yaml +0 -39
- data/templates/steps/bootstrap/fedora/liveos_bootstrap.yaml +0 -123
- data/templates/steps/bootstrap/initialize_disk_qemu.yaml +0 -72
- data/templates/steps/bootstrap/install_bootloader.yaml +0 -42
- data/templates/steps/bootstrap/ubuntu/debootstrap.yaml +0 -27
- data/templates/steps/export/save_appliance.yaml +0 -58
- data/templates/steps/export/save_vagrant_box.yaml +0 -29
- data/templates/vagrant-debian7.yaml +0 -31
data/lib/kameleon/environment.rb
CHANGED
@@ -7,30 +7,35 @@ module Kameleon
|
|
7
7
|
attr_accessor :workspace
|
8
8
|
attr_accessor :templates_path
|
9
9
|
attr_accessor :build_path
|
10
|
-
attr_accessor :
|
10
|
+
attr_accessor :cache_path
|
11
11
|
attr_accessor :debug
|
12
12
|
|
13
|
+
def script?
|
14
|
+
@script
|
15
|
+
end
|
13
16
|
|
14
17
|
def initialize(options = {})
|
15
|
-
@logger = Log4r::Logger.new("kameleon::[kameleon]")
|
16
18
|
# symbolify commandline options
|
17
19
|
options = options.inject({}) {|result,(key,value)| result.update({key.to_sym => value})}
|
18
20
|
workspace = File.expand_path(Dir.pwd)
|
19
21
|
build_path = File.expand_path(options[:build_path] || File.join(workspace, "build"))
|
22
|
+
cache_path = File.expand_path(options[:cache_path] || File.join(build_path, "cache"))
|
20
23
|
defaults = {
|
21
24
|
:workspace => Pathname.new(workspace),
|
22
25
|
:templates_path => Kameleon.templates_path,
|
23
26
|
:templates_names => Kameleon.templates_names,
|
24
27
|
:build_path => Pathname.new(build_path),
|
25
|
-
:
|
28
|
+
:cache_path => Pathname.new(cache_path),
|
29
|
+
:script => options[:script],
|
26
30
|
}
|
27
31
|
options = defaults.merge(options)
|
28
|
-
|
32
|
+
Kameleon.ui.debug("Environment initialized (#{self})")
|
29
33
|
# Injecting all variables of the options and assign the variables
|
30
34
|
options.each do |key, value|
|
31
35
|
instance_variable_set("@#{key}".to_sym, options[key])
|
32
|
-
|
36
|
+
Kameleon.ui.debug(" @#{key} : #{options[key]}")
|
33
37
|
end
|
38
|
+
@debug = true if ENV['KAMELEON_DEBUG']
|
34
39
|
end
|
35
40
|
end
|
36
41
|
end
|
data/lib/kameleon/error.rb
CHANGED
@@ -1,4 +1,3 @@
|
|
1
|
-
require 'thor/error'
|
2
1
|
|
3
2
|
module Kameleon
|
4
3
|
class Error < ::StandardError
|
@@ -14,6 +13,7 @@ module Kameleon
|
|
14
13
|
end
|
15
14
|
end
|
16
15
|
|
16
|
+
class Exit < Error; status_code(0) ; end
|
17
17
|
class KameleonError < Error; status_code(1) ; end
|
18
18
|
class ExecError < Error; status_code(2) ; end
|
19
19
|
class InternalError < Error; status_code(3) ; end
|
@@ -23,35 +23,4 @@ module Kameleon
|
|
23
23
|
class BuildError < Error; status_code(7) ; end
|
24
24
|
class AbortError < Error; status_code(8) ; end
|
25
25
|
class TemplateNotFound < Error; status_code(9) ; end
|
26
|
-
|
27
|
-
def self.with_friendly_errors
|
28
|
-
yield
|
29
|
-
rescue Kameleon::Error => e
|
30
|
-
e.message.split( /\r?\n/ ).each {|m| Kameleon.logger.fatal m }
|
31
|
-
exit e.status_code
|
32
|
-
rescue Thor::UndefinedTaskError => e
|
33
|
-
$stderr << "#{e.message}\n"
|
34
|
-
e.backtrace.each {|m| Kameleon.logger.debug m }
|
35
|
-
exit 15
|
36
|
-
rescue Thor::Error => e
|
37
|
-
$stderr << "#{e.message}\n"
|
38
|
-
e.backtrace.each {|m| Kameleon.logger.debug m }
|
39
|
-
exit 15
|
40
|
-
rescue SystemExit, Interrupt => e
|
41
|
-
Kameleon.logger.fatal("Quitting...")
|
42
|
-
exit 1
|
43
|
-
rescue Errno::ENOENT => e
|
44
|
-
$stderr << "#{e.message}\n"
|
45
|
-
e.backtrace.each {|m| Kameleon.logger.debug m }
|
46
|
-
exit 16
|
47
|
-
rescue Exception => e
|
48
|
-
if ENV["KAMELEON_LOG"] != "debug"
|
49
|
-
$stderr << "Unfortunately, a fatal error has occurred : "\
|
50
|
-
"#{e.message}.\nUse --debug option for more details\n"
|
51
|
-
else
|
52
|
-
Kameleon.logger.debug "Error : #{e}"
|
53
|
-
e.backtrace.each {|m| puts "==> #{m}" }
|
54
|
-
end
|
55
|
-
exit 666
|
56
|
-
end
|
57
26
|
end
|
@@ -1,7 +1,6 @@
|
|
1
1
|
require 'childprocess'
|
2
2
|
require 'singleton'
|
3
3
|
require 'socket'
|
4
|
-
|
5
4
|
module Kameleon
|
6
5
|
#This ruby class will control the execution of Polipo web proxy
|
7
6
|
class Persistent_cache
|
@@ -12,11 +11,15 @@ module Kameleon
|
|
12
11
|
attr_reader :polipo_port
|
13
12
|
attr_writer :activated
|
14
13
|
attr_reader :cwd
|
15
|
-
|
14
|
+
attr_writer :polipo_path
|
16
15
|
attr_reader :name
|
16
|
+
attr_writer :cache_path
|
17
|
+
attr_accessor :mode
|
18
|
+
attr_accessor :name
|
19
|
+
attr_accessor :recipe_files # have to check those.
|
20
|
+
attr_accessor :recipe_path
|
17
21
|
|
18
22
|
def initialize()
|
19
|
-
@logger = Log4r::Logger.new("kameleon::[kameleon]")
|
20
23
|
## we must configure Polipo to be execute for the in and out context
|
21
24
|
## we have to start polipo in the out context for debootstrap step
|
22
25
|
|
@@ -28,18 +31,28 @@ module Kameleon
|
|
28
31
|
@polipo_port = find_unused_port
|
29
32
|
|
30
33
|
@polipo_cmd_options = {:diskCacheRoot => "",
|
31
|
-
|
32
|
-
|
33
|
-
|
34
|
-
|
34
|
+
:idleTime => "1",
|
35
|
+
:chunkHighMark => "425165824",
|
36
|
+
:proxyPort => @polipo_port,
|
37
|
+
:relaxTransparency =>"true",
|
38
|
+
:daemonise => false,
|
39
|
+
:logFile => File.join(Kameleon.env.build_path, 'polipo.log')
|
35
40
|
}
|
36
41
|
|
37
42
|
@activated = false
|
38
43
|
|
39
|
-
@
|
44
|
+
@mode = nil #It could be build or from
|
45
|
+
@cache_dir = Kameleon.env.cache_path
|
40
46
|
@polipo_path = nil
|
41
47
|
@cwd = ""
|
42
|
-
|
48
|
+
#structure {:cmd => "cmd", :stdout_filename => "file_name"}
|
49
|
+
@cmd_cached = []
|
50
|
+
@cache_path = ""
|
51
|
+
@current_cmd_id = nil
|
52
|
+
@current_step_dir = nil
|
53
|
+
@recipe_file = nil
|
54
|
+
@steps_files = []
|
55
|
+
@cached_recipe_dir = nil
|
43
56
|
end
|
44
57
|
|
45
58
|
def find_unused_port
|
@@ -49,7 +62,7 @@ module Kameleon
|
|
49
62
|
ports.each do |p|
|
50
63
|
begin
|
51
64
|
port = p
|
52
|
-
tmp = TCPServer.new('localhost',port)
|
65
|
+
tmp = TCPServer.new('localhost', port)
|
53
66
|
rescue
|
54
67
|
port =0
|
55
68
|
end
|
@@ -61,12 +74,11 @@ module Kameleon
|
|
61
74
|
|
62
75
|
def check_polipo_binary
|
63
76
|
|
64
|
-
|
65
77
|
@polipo_path ||= which("polipo")
|
66
78
|
|
67
79
|
if @polipo_path.nil? then
|
68
|
-
|
69
|
-
|
80
|
+
Kameleon.ui.error("Polipo binary not found, make sure it is in your current PATH")
|
81
|
+
Kameleon.ui.error("or use the option --proxy-path")
|
70
82
|
raise BuildError, "Failed to use persistent cache"
|
71
83
|
end
|
72
84
|
end
|
@@ -75,54 +87,131 @@ module Kameleon
|
|
75
87
|
@activated
|
76
88
|
end
|
77
89
|
|
78
|
-
|
79
90
|
def cwd=(dir)
|
80
91
|
@cwd = dir
|
81
|
-
@cache_dir = @cwd + "/cache/"
|
82
92
|
end
|
83
93
|
|
84
94
|
def create_cache_directory(step_name)
|
85
|
-
|
86
|
-
directory_name = @cache_dir
|
95
|
+
Kameleon.ui.debug("Creating cache directory #{step_name} for Polipo")
|
96
|
+
directory_name = File.join(@cache_dir,"#{step_name}")
|
87
97
|
FileUtils.mkdir_p directory_name
|
88
98
|
directory_name
|
89
99
|
end
|
90
100
|
|
91
101
|
def start_web_proxy_in(directory)
|
102
|
+
|
103
|
+
## setting current step dir
|
104
|
+
@current_step_dir = directory
|
92
105
|
## This function assumes that the cache directory has already been created by the engine
|
93
106
|
## Stopping first the previous proxy
|
94
107
|
## have to check if polipo is running
|
95
|
-
|
96
|
-
@polipo_process.stop unless @polipo_process.nil?
|
97
|
-
command = ["#{@polipo_path}/polipo"]
|
108
|
+
Kameleon.ui.debug("Starting web proxy Polipo in directory #{directory} using port: #{@polipo_port}")
|
109
|
+
@polipo_process.stop(0) unless @polipo_process.nil?
|
110
|
+
command = ["#{@polipo_path}/polipo", "-c", "/dev/null"]
|
98
111
|
@polipo_cmd_options[:diskCacheRoot] = directory
|
99
112
|
@polipo_cmd_options.each{ |v,k| command.push("#{v}=#{k}") }
|
100
113
|
ChildProcess.posix_spawn = true
|
114
|
+
Kameleon.ui.debug("Starting process '#{command}'")
|
101
115
|
@polipo_process = ChildProcess.build(*command)
|
102
|
-
@polipo_process.io.stdout = Tempfile.new("polipo_output")
|
103
116
|
@polipo_process.start
|
117
|
+
return true
|
104
118
|
end
|
105
119
|
|
106
120
|
|
107
121
|
def stop_web_proxy
|
108
122
|
@polipo_process.stop
|
109
|
-
|
123
|
+
Kameleon.ui.info("Stopping web proxy polipo")
|
110
124
|
end
|
111
125
|
|
112
126
|
def pack()
|
113
|
-
|
114
|
-
execute("tar","-cf #{@name}-cache.tar cache/",@cwd)
|
115
|
-
# The cache directory cannot be deleted due to the checkpoints
|
127
|
+
Kameleon.ui.info("Packing up the generated cache in #{@cwd}/#{@name}-cache.tar")
|
128
|
+
execute("tar","-cf #{@name}-cache.tar -C cache/ .",@cwd)
|
116
129
|
end
|
117
130
|
|
118
131
|
def unpack(cache_path)
|
119
|
-
|
120
|
-
|
132
|
+
Kameleon.ui.info("Unpacking persistent cache: #{cache_path}")
|
133
|
+
FileUtils.mkdir_p @cache_dir
|
134
|
+
execute("tar","-xf #{cache_path} -C #{@cache_dir}")
|
135
|
+
end
|
136
|
+
|
137
|
+
|
138
|
+
# This function caches the command with its respective stdout
|
139
|
+
# a command id is associate to a file
|
140
|
+
def cache_cmd_id(cmd_identifier)
|
141
|
+
@current_cmd_id = cmd_identifier
|
142
|
+
return true
|
143
|
+
end
|
144
|
+
|
145
|
+
def cache_cmd(cmd,file_path)
|
146
|
+
Kameleon.ui.info("Caching file")
|
147
|
+
Kameleon.ui.debug("command: cp #{file_path} #{@cwd}/cache/files/")
|
148
|
+
FileUtils.mkdir_p @current_step_dir + "/data/"
|
149
|
+
FileUtils.cp file_path, @current_step_dir + "/data/"
|
150
|
+
@cmd_cached.push({:cmd_id => @current_cmd_id,
|
151
|
+
:cmd => cmd ,
|
152
|
+
:stdout_filename => File.basename(file_path)})
|
153
|
+
end
|
154
|
+
|
155
|
+
def get_cache_cmd(cmd)
|
156
|
+
return false if @mode == :build
|
157
|
+
cache_line = @cmd_cached.select{ |reg|
|
158
|
+
(reg[:cmd_id] == @current_cmd_id && reg[:cmd] == cmd) }.first
|
159
|
+
|
160
|
+
return File.new("#{@current_step_dir}/data/#{cache_line[:stdout_filename]}","r")
|
161
|
+
end
|
162
|
+
|
163
|
+
def stop()
|
164
|
+
@polipo_process.stop
|
165
|
+
Kameleon.ui.info("Stopping web proxy polipo")
|
166
|
+
Kameleon.ui.info("Finishing persistent cache with last files")
|
167
|
+
|
168
|
+
if @mode == :build then
|
169
|
+
File.open("#{@cache_dir}/cache_cmd_index",'w+') do |f|
|
170
|
+
f.puts(@cmd_cached.to_yaml)
|
171
|
+
end
|
172
|
+
|
173
|
+
unless @recipe_files.empty?
|
174
|
+
recipe_dir = Pathname.new(common_prefix(@recipe_files))
|
175
|
+
all_files = @recipe_files.push(@recipe_path)
|
176
|
+
Kameleon::Utils.copy_files(recipe_dir, @cache_dir, all_files)
|
177
|
+
end
|
178
|
+
## Saving metadata information
|
179
|
+
Kameleon.ui.info("Caching recipe")
|
180
|
+
File.open("#{@cached_recipe_dir}/header",'w+') do |f|
|
181
|
+
f.puts({:recipe_path => @recipe_path.to_s}.to_yaml)
|
182
|
+
end
|
183
|
+
pack
|
184
|
+
end
|
185
|
+
end
|
186
|
+
|
187
|
+
def start()
|
188
|
+
check_polipo_binary
|
189
|
+
if @mode == :from then
|
190
|
+
begin
|
191
|
+
unpack(@cache_path)
|
192
|
+
rescue
|
193
|
+
raise BuildError, "Failed to untar the persistent cache file"
|
194
|
+
end
|
195
|
+
## We have to load the file
|
196
|
+
@cmd_cached = YAML.load(File.read("#{@cache_dir}/cache_cmd_index"))
|
197
|
+
end
|
198
|
+
@activated = true
|
199
|
+
@cached_recipe_dir = @cache_dir
|
200
|
+
FileUtils.mkdir_p @cached_recipe_dir
|
201
|
+
end
|
202
|
+
|
203
|
+
def get_recipe()
|
204
|
+
cached_recipe=Dir.mktmpdir("cache")
|
205
|
+
puts "cache path : #{@cache_path}"
|
206
|
+
execute("tar","-xf #{@cache_path} -C #{cached_recipe} .")
|
207
|
+
Kameleon.ui.info("Getting cached recipe")
|
208
|
+
recipe_header = YAML::load(File.read("#{cached_recipe}/header"))
|
209
|
+
recipe_file = recipe_header[:recipe_path]
|
210
|
+
return recipe_file
|
121
211
|
end
|
122
212
|
|
123
213
|
def execute(cmd,args,dir=nil)
|
124
214
|
command = [cmd ] + args.split(" ")
|
125
|
-
# @logger.notice(" command generated: #{command}")
|
126
215
|
process = ChildProcess.build(*command)
|
127
216
|
process.cwd = dir unless dir.nil?
|
128
217
|
process.start
|
@@ -137,8 +226,17 @@ module Kameleon
|
|
137
226
|
return nil
|
138
227
|
end
|
139
228
|
|
229
|
+
def common_prefix(paths)
|
230
|
+
return '' if paths.empty?
|
231
|
+
return paths.first.split('/').slice(0...-1).join('/') if paths.length <= 1
|
232
|
+
arr = paths.sort
|
233
|
+
first = arr.first.to_s.split('/')
|
234
|
+
last = arr.last.to_s.split('/')
|
235
|
+
i = 0
|
236
|
+
i += 1 while first[i] == last[i] && i <= first.length
|
237
|
+
first.slice(0, i).join('/')
|
238
|
+
end
|
140
239
|
|
141
240
|
end
|
142
241
|
|
143
242
|
end
|
144
|
-
|
data/lib/kameleon/recipe.rb
CHANGED
@@ -13,9 +13,11 @@ module Kameleon
|
|
13
13
|
attr_accessor :checkpoint
|
14
14
|
attr_accessor :checkpoint_path
|
15
15
|
attr_accessor :metainfo
|
16
|
+
attr_accessor :files
|
17
|
+
attr_accessor :base_recipes_files
|
18
|
+
|
16
19
|
|
17
20
|
def initialize(path, kwargs = {})
|
18
|
-
@logger = Log4r::Logger.new("kameleon::[kameleon]")
|
19
21
|
@path = Pathname.new(File.expand_path(path))
|
20
22
|
@name = (@path.basename ".yaml").to_s
|
21
23
|
@recipe_content = File.open(@path, 'r') { |f| f.read }
|
@@ -24,27 +26,24 @@ module Kameleon
|
|
24
26
|
"setup" => Section.new("setup"),
|
25
27
|
"export" => Section.new("export"),
|
26
28
|
}
|
27
|
-
kameleon_id = SecureRandom.uuid
|
28
29
|
@global = {
|
29
30
|
"kameleon_recipe_name" => @name,
|
30
31
|
"kameleon_recipe_dir" => File.dirname(@path),
|
31
|
-
"kameleon_uuid" => kameleon_id,
|
32
|
-
"kameleon_short_uuid" => kameleon_id.split("-").last,
|
33
32
|
"kameleon_cwd" => File.join(Kameleon.env.build_path, @name),
|
34
|
-
"in_context" => {"cmd"=> "/bin/bash"},
|
35
|
-
"out_context" => {"cmd"=> "/bin/bash"}
|
33
|
+
"in_context" => {"cmd" => "/bin/bash", "proxy_cache" => "localhost"},
|
34
|
+
"out_context" => {"cmd" => "/bin/bash", "proxy_cache" => "localhost"}
|
36
35
|
}
|
37
36
|
@aliases = {}
|
38
37
|
@checkpoint = nil
|
39
38
|
@files = []
|
40
|
-
|
39
|
+
Kameleon.ui.debug("Initialize new recipe (#{path})")
|
41
40
|
@base_recipes_files = [@path]
|
42
|
-
load!
|
41
|
+
load! :strict => false
|
43
42
|
end
|
44
43
|
|
45
44
|
def load!(kwargs = {})
|
46
45
|
# Find recipe path
|
47
|
-
|
46
|
+
Kameleon.ui.debug("Loading #{@path}")
|
48
47
|
fail RecipeError, "Could not find this following recipe : #{@path}" \
|
49
48
|
unless File.file? @path
|
50
49
|
yaml_recipe = YAML.load File.open @path
|
@@ -59,22 +58,27 @@ module Kameleon
|
|
59
58
|
@global.merge!(yaml_recipe.fetch("global", {}))
|
60
59
|
# Resolve dynamically-defined variables !!
|
61
60
|
resolved_global = Utils.resolve_vars(@global.to_yaml, @path, @global, kwargs)
|
62
|
-
@global.merge
|
61
|
+
resolved_global = @global.merge YAML.load(resolved_global)
|
63
62
|
# Loads aliases
|
64
63
|
load_aliases(yaml_recipe)
|
65
64
|
# Loads checkpoint configuration
|
66
65
|
load_checkpoint_config(yaml_recipe)
|
67
66
|
|
68
67
|
#Find and load steps
|
69
|
-
|
70
|
-
|
71
|
-
|
72
|
-
|
73
|
-
|
68
|
+
steps_dirs = [
|
69
|
+
File.join(File.dirname(@path), 'steps'),
|
70
|
+
File.expand_path(File.join(File.dirname(@path), '..', 'steps')),
|
71
|
+
]
|
72
|
+
resolved_global['include_steps'] ||= []
|
73
|
+
resolved_global['include_steps'].push ''
|
74
|
+
resolved_global['include_steps'].flatten!
|
75
|
+
resolved_global['include_steps'].compact!
|
74
76
|
@sections.values.each do |section|
|
75
|
-
dir_to_search =
|
76
|
-
|
77
|
-
File.join(steps_dir, path)
|
77
|
+
dir_to_search = resolved_global['include_steps'].map do |path|
|
78
|
+
steps_dirs.map do |steps_dir|
|
79
|
+
[File.join(steps_dir, section.name, path),
|
80
|
+
File.join(steps_dir, path)]
|
81
|
+
end
|
78
82
|
end
|
79
83
|
dir_to_search.flatten!
|
80
84
|
if yaml_recipe.key? section.name
|
@@ -104,7 +108,7 @@ module Kameleon
|
|
104
108
|
end
|
105
109
|
end
|
106
110
|
if embedded_step
|
107
|
-
|
111
|
+
Kameleon.ui.debug("Loading embedded macrostep #{name}")
|
108
112
|
macrostep = load_macrostep(nil, name, args)
|
109
113
|
section.macrosteps.push(macrostep)
|
110
114
|
next
|
@@ -114,16 +118,16 @@ module Kameleon
|
|
114
118
|
dir_to_search.each do |dir|
|
115
119
|
macrostep_path = Pathname.new(File.join(dir, name + '.yaml'))
|
116
120
|
if File.file?(macrostep_path)
|
117
|
-
|
121
|
+
Kameleon.ui.debug("Loading macrostep #{macrostep_path}")
|
118
122
|
macrostep = load_macrostep(macrostep_path, name, args)
|
119
123
|
section.macrosteps.push(macrostep)
|
120
124
|
@files.push(macrostep_path)
|
121
|
-
|
125
|
+
Kameleon.ui.debug("Macrostep '#{name}' found in this path: " \
|
122
126
|
"#{macrostep_path}")
|
123
127
|
loaded = true
|
124
128
|
break
|
125
129
|
else
|
126
|
-
|
130
|
+
Kameleon.ui.debug("Macrostep '#{name}' not found in this path: " \
|
127
131
|
"#{macrostep_path}")
|
128
132
|
end
|
129
133
|
end
|
@@ -131,7 +135,7 @@ module Kameleon
|
|
131
135
|
end
|
132
136
|
end
|
133
137
|
end
|
134
|
-
|
138
|
+
Kameleon.ui.debug("Loading recipe metadata")
|
135
139
|
@metainfo = {
|
136
140
|
"description" => Utils.extract_meta_var("description", @recipe_content)
|
137
141
|
}
|
@@ -194,12 +198,13 @@ module Kameleon
|
|
194
198
|
elsif aliases.kind_of? String
|
195
199
|
dir_search = [
|
196
200
|
File.join(File.dirname(@path), "steps", "aliases"),
|
201
|
+
File.join(File.dirname(@path), "..", "steps", "aliases"),
|
197
202
|
File.join(File.dirname(@path), "aliases")
|
198
203
|
]
|
199
204
|
dir_search.each do |dir_path|
|
200
205
|
path = Pathname.new(File.join(dir_path, aliases))
|
201
206
|
if File.file?(path)
|
202
|
-
|
207
|
+
Kameleon.ui.debug("Loading aliases #{path}")
|
203
208
|
@aliases = YAML.load_file(path)
|
204
209
|
@files.push(path)
|
205
210
|
return path
|
@@ -219,20 +224,25 @@ module Kameleon
|
|
219
224
|
elsif checkpoint.kind_of? String
|
220
225
|
dir_search = [
|
221
226
|
File.join(File.dirname(@path), "steps", "checkpoints"),
|
227
|
+
File.join(File.dirname(@path), "..", "steps", "checkpoints"),
|
222
228
|
File.join(File.dirname(@path), "checkpoints")
|
223
229
|
]
|
224
230
|
dir_search.each do |dir_path|
|
225
231
|
path = Pathname.new(File.join(dir_path, checkpoint))
|
226
232
|
if File.file?(path)
|
227
|
-
|
233
|
+
Kameleon.ui.debug("Loading checkpoint configuration #{path}")
|
228
234
|
@checkpoint = YAML.load_file(path)
|
229
235
|
@checkpoint["path"] = path.to_s
|
230
236
|
@files.push(path)
|
231
|
-
return path
|
232
237
|
end
|
233
238
|
end
|
234
239
|
fail RecipeError, "Checkpoint configuraiton file '#{path}' " \
|
235
|
-
"does not exists"
|
240
|
+
"does not exists" if @checkpoint.nil?
|
241
|
+
end
|
242
|
+
(@checkpoint.keys - ["path"]).each do |key|
|
243
|
+
@checkpoint[key].map! do |cmd|
|
244
|
+
Kameleon::Command.new(cmd, "checkpoint")
|
245
|
+
end
|
236
246
|
end
|
237
247
|
end
|
238
248
|
end
|
@@ -295,15 +305,21 @@ module Kameleon
|
|
295
305
|
end
|
296
306
|
|
297
307
|
def find_microstep(microstep_name, loaded_microsteps)
|
298
|
-
|
308
|
+
Kameleon.ui.debug("Looking for microstep #{microstep_name}")
|
299
309
|
loaded_microsteps.each do |microstep|
|
300
310
|
if microstep_name.eql? microstep.name
|
301
311
|
return microstep
|
302
312
|
end
|
303
313
|
end
|
314
|
+
return nil
|
304
315
|
end
|
305
316
|
|
306
317
|
def resolve!
|
318
|
+
unless @global.keys.include? "kameleon_uuid"
|
319
|
+
kameleon_id = SecureRandom.uuid
|
320
|
+
@global["kameleon_uuid"] = kameleon_id
|
321
|
+
@global["kameleon_short_uuid"] = kameleon_id.split("-").last
|
322
|
+
end
|
307
323
|
# Resolve dynamically-defined variables !!
|
308
324
|
resolved_global = Utils.resolve_vars(@global.to_yaml, @path, @global)
|
309
325
|
@global.merge! YAML.load(resolved_global)
|
@@ -311,7 +327,7 @@ module Kameleon
|
|
311
327
|
consistency_check
|
312
328
|
resolve_checkpoint unless @checkpoint.nil?
|
313
329
|
|
314
|
-
|
330
|
+
Kameleon.ui.info("Resolving variables")
|
315
331
|
@sections.values.each do |section|
|
316
332
|
section.macrosteps.each do |macrostep|
|
317
333
|
macrostep.resolve_variables!(@global)
|
@@ -321,7 +337,7 @@ module Kameleon
|
|
321
337
|
@sections.values.each do |section|
|
322
338
|
section.macrosteps.each do |macrostep|
|
323
339
|
# First pass : resolve aliases
|
324
|
-
|
340
|
+
Kameleon.ui.debug("Resolving aliases for macrostep '#{macrostep.name}'")
|
325
341
|
macrostep.microsteps.each do |microstep|
|
326
342
|
microstep.commands.map! do |cmd|
|
327
343
|
# resolve alias
|
@@ -329,7 +345,7 @@ module Kameleon
|
|
329
345
|
end
|
330
346
|
end
|
331
347
|
# flatten for multiple-command alias + variables
|
332
|
-
|
348
|
+
Kameleon.ui.debug("Resolving check statements for macrostep '#{macrostep.name}'")
|
333
349
|
macrostep.microsteps.each { |microstep| microstep.commands.flatten! }
|
334
350
|
# Second pass : resolve variables + clean/init hooks
|
335
351
|
macrostep.microsteps.each do |microstep|
|
@@ -337,7 +353,7 @@ module Kameleon
|
|
337
353
|
resolve_hooks(cmd, macrostep, microstep)
|
338
354
|
end
|
339
355
|
end
|
340
|
-
|
356
|
+
Kameleon.ui.debug("Compacting macrostep '#{macrostep.name}'")
|
341
357
|
# remove empty steps
|
342
358
|
macrostep.microsteps.map! do |microstep|
|
343
359
|
microstep.commands.compact!
|
@@ -345,7 +361,7 @@ module Kameleon
|
|
345
361
|
end
|
346
362
|
# remove nil values
|
347
363
|
macrostep.microsteps.compact!
|
348
|
-
|
364
|
+
Kameleon.ui.debug("Resolving commands for macrostep '#{macrostep.name}'")
|
349
365
|
macrostep.microsteps.each do |microstep|
|
350
366
|
microstep.resolve!
|
351
367
|
end
|
@@ -365,7 +381,7 @@ module Kameleon
|
|
365
381
|
end
|
366
382
|
end
|
367
383
|
end
|
368
|
-
|
384
|
+
Kameleon.ui.info("Starting recipe consistency check")
|
369
385
|
# check context args
|
370
386
|
required_args = %w(cmd)
|
371
387
|
missings = []
|
@@ -385,10 +401,12 @@ module Kameleon
|
|
385
401
|
end
|
386
402
|
|
387
403
|
def resolve_checkpoint()
|
388
|
-
|
389
|
-
@checkpoint[key]
|
404
|
+
(@checkpoint.keys - ["path"]).each do |key|
|
405
|
+
@checkpoint[key].each do |cmd|
|
406
|
+
cmd.string_cmd = Utils.resolve_vars(cmd.string_cmd,
|
390
407
|
@checkpoint["path"],
|
391
408
|
@global)
|
409
|
+
end
|
392
410
|
end
|
393
411
|
end
|
394
412
|
|
@@ -490,17 +508,21 @@ module Kameleon
|
|
490
508
|
end
|
491
509
|
|
492
510
|
def calculate_step_identifiers
|
493
|
-
|
511
|
+
Kameleon.ui.debug("Calculating microstep identifiers")
|
494
512
|
base_salt = ""
|
495
513
|
order = 0
|
496
514
|
@sections.values.each do |section|
|
497
515
|
section.sequence do |macrostep|
|
498
516
|
macrostep.sequence do |microstep|
|
499
|
-
|
517
|
+
if ["redo", "skip"].include? microstep.on_checkpoint
|
518
|
+
microstep.calculate_identifier ""
|
519
|
+
else
|
520
|
+
base_salt = microstep.calculate_identifier base_salt
|
521
|
+
end
|
500
522
|
slug = "#{section.name}/#{macrostep.name}/#{microstep.name}"
|
501
523
|
microstep.slug = slug
|
502
524
|
microstep.order = (order += 1)
|
503
|
-
|
525
|
+
Kameleon.ui.debug(" #{microstep.slug}: #{microstep.identifier}")
|
504
526
|
end
|
505
527
|
end
|
506
528
|
end
|
@@ -531,73 +553,8 @@ module Kameleon
|
|
531
553
|
end
|
532
554
|
|
533
555
|
class RecipeTemplate < Recipe
|
534
|
-
|
535
|
-
|
536
|
-
while true
|
537
|
-
@logger.progress_notice msg
|
538
|
-
answer = $stdin.gets.downcase
|
539
|
-
raise AbortError, "Execution aborted..." if answer.nil?
|
540
|
-
answer.chomp!
|
541
|
-
if ["y", "n" , "", "a"].include?(answer)
|
542
|
-
if ["y", ""].include? answer
|
543
|
-
return true
|
544
|
-
elsif answer.eql? "a"
|
545
|
-
raise AbortError, "Aborted..."
|
546
|
-
end
|
547
|
-
return false
|
548
|
-
end
|
549
|
-
end
|
550
|
-
end
|
551
|
-
|
552
|
-
def safe_copy_file(src, dst, force)
|
553
|
-
if File.exists? dst
|
554
|
-
diff = Diffy::Diff.new(dst.to_s, src.to_s, :source => "files").to_s
|
555
|
-
unless diff.chomp.empty?
|
556
|
-
@logger.notice("conflict #{dst}")
|
557
|
-
@logger.notice("Differences between the old and the new :")
|
558
|
-
puts Diffy::Diff.new(dst.to_s, src.to_s,
|
559
|
-
:source => "files",
|
560
|
-
:context => 1,
|
561
|
-
:include_diff_info => true).to_s
|
562
|
-
msg = "Overwrite #{dst}? [Y]es/[n]o/[a]bort : "
|
563
|
-
if force || get_answer(msg)
|
564
|
-
FileUtils.copy_file(src, dst)
|
565
|
-
end
|
566
|
-
else
|
567
|
-
@logger.notice("identical #{dst}")
|
568
|
-
end
|
569
|
-
else
|
570
|
-
unless File.dirname(dst).eql? "/"
|
571
|
-
FileUtils.mkdir_p File.dirname(dst)
|
572
|
-
end
|
573
|
-
@logger.notice("create #{dst}")
|
574
|
-
FileUtils.copy_file(src, dst)
|
575
|
-
end
|
576
|
-
end
|
577
|
-
|
578
|
-
def copy_extended_recipe(recipe_name, force)
|
579
|
-
Dir::mktmpdir do |tmp_dir|
|
580
|
-
recipe_path = File.join(tmp_dir, recipe_name + '.yaml')
|
581
|
-
## copying recipe
|
582
|
-
File.open(recipe_path, 'w+') do |file|
|
583
|
-
extend_erb_tpl = File.join(Kameleon.env.templates_path, "extend.erb")
|
584
|
-
tpl = ERB.new(File.open(extend_erb_tpl, 'rb') { |f| f.read })
|
585
|
-
result = tpl.result(binding)
|
586
|
-
file.write(result)
|
587
|
-
end
|
588
|
-
recipe_dst = File.join(Kameleon.env.workspace, recipe_name + '.yaml')
|
589
|
-
safe_copy_file(recipe_path, Pathname.new(recipe_dst), force)
|
590
|
-
end
|
591
|
-
end
|
592
|
-
|
593
|
-
def copy_template(force)
|
594
|
-
## copying steps
|
595
|
-
files2copy = @base_recipes_files + @files
|
596
|
-
files2copy.each do |path|
|
597
|
-
relative_path = path.relative_path_from(Kameleon.env.templates_path)
|
598
|
-
dst = File.join(Kameleon.env.workspace, relative_path)
|
599
|
-
safe_copy_file(path, dst, force)
|
600
|
-
end
|
556
|
+
def relative_path()
|
557
|
+
@path.relative_path_from(Kameleon.env.templates_path)
|
601
558
|
end
|
602
559
|
end
|
603
560
|
end
|