rbbt-image 0.1.38 → 0.1.40
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/Dockerfile +3 -3
- data/bin/build_rbbt_provision_sh.rb +123 -245
- data/share/provision_scripts/R.sh +66 -0
- data/share/provision_scripts/{R_setup.sh → R_custom.sh} +1 -1
- data/share/provision_scripts/alpine_basic_setup.sh +17 -0
- data/share/provision_scripts/alpine_extended_setup.sh +47 -0
- data/share/provision_scripts/alpine_setup.sh +20 -0
- data/share/provision_scripts/bootstrap.sh +16 -4
- data/share/provision_scripts/functions.sh +58 -0
- data/share/provision_scripts/gem.sh +42 -0
- data/share/provision_scripts/hacks.sh +1 -1
- data/share/provision_scripts/java.sh +3 -0
- data/share/provision_scripts/migrate.sh +6 -0
- data/share/provision_scripts/python.sh +4 -0
- data/share/provision_scripts/{python_setup.sh → python_custom.sh} +7 -1
- data/share/provision_scripts/{ruby_setup.sh → ruby_custom.sh} +1 -1
- data/share/provision_scripts/slurm_loopback.sh +45 -0
- data/share/provision_scripts/{tokyocabinet_setup.sh → tokyocabinet.sh} +7 -2
- data/share/provision_scripts/ubuntu_setup.sh +65 -65
- data/share/provision_scripts/variables.sh +10 -12
- metadata +19 -11
- data/share/provision_scripts/R_packages.sh +0 -51
- data/share/provision_scripts/gem_setup.sh +0 -46
- /data/share/provision_scripts/{perl_setup.sh → perl_custom.sh} +0 -0
- /data/share/provision_scripts/{user_setup.sh → user.sh} +0 -0
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: db006b39f539ccfe01887fe76863904209abc11ab2caf2c53d9b77b35f463662
|
|
4
|
+
data.tar.gz: efdead2dfa7cd0d79303524e415cf1acdcc3db2cdeb1aa562e268d44e9f4a372
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: b481e7a4daf5a2f4155959d7e2b53aeee8d8cb2da95ba252df5f590407679bfe4b5faf70f736ffd735b8bf1cc08717fb8b80471042649a9a4ffc8a2aafd25b67
|
|
7
|
+
data.tar.gz: bb5db5c1d64cecdd49e8bb8e1557c296e2a29e36be7ce3fa3c12644e40448157456b988ed5c4ab1a5e170f14bfec4978768422444b2206a239bbcd0fce84cea1
|
data/Dockerfile
CHANGED
|
@@ -25,18 +25,10 @@ $ #{$0} [options]
|
|
|
25
25
|
-ws--workflow_server* Rbbt remote workflow server
|
|
26
26
|
-rr--remote_resources* Remote resources to gather from file-server
|
|
27
27
|
-rw--remote_workflows* Remote workflows server from workflow-server
|
|
28
|
-
-
|
|
29
|
-
-st--skip_tokyocabinet Skip tokyocabinet setup installation
|
|
30
|
-
-sr--skip_ruby Skip ruby setup installation
|
|
31
|
-
-sp--skip_perl Skip perl setup installation
|
|
32
|
-
-spy--skip_python Skip python setup installation
|
|
33
|
-
-sg--skip_gem Skip ruby gem installation
|
|
34
|
-
-sR--skip_R Skip R setup
|
|
35
|
-
-su--skip_user_setup Skip user setup
|
|
36
|
-
-sb--skip_bootstrap Skip user bootstrap
|
|
37
|
-
-Rc--R_custom Install a custom installation of R
|
|
38
|
-
-Rp--R_packages Install basic R packages
|
|
28
|
+
-bs--base_system* Version of base system initialization script to use (default: ubuntu)
|
|
39
29
|
-Rbv--ruby_version* Ruby version to use, using three numbers (defaults to 2.4.1)
|
|
30
|
+
-d--do* List of steps to do
|
|
31
|
+
-nd--not_do* List of steps to not do
|
|
40
32
|
-op--optimize Optimize files under ~/.rbbt
|
|
41
33
|
-dep--container_dependency* Use a different image in Dockerfile, Singularity and Virtualbox
|
|
42
34
|
-dt--docker* Build docker image using the provided name
|
|
@@ -66,27 +58,26 @@ script_dir = File.join(root_dir, "share/provision_scripts/")
|
|
|
66
58
|
# options[:skip_bootstrap] = true
|
|
67
59
|
#end
|
|
68
60
|
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
61
|
+
all_steps = %w(functions base_system tokyocabinet ruby_custom gem java R_custom R perl_custom python_custom python user slurm_loopback hacks)
|
|
62
|
+
|
|
63
|
+
|
|
64
|
+
do_steps = options.include?("do")? (all_steps & options[:do].split(",")) : all_steps
|
|
65
|
+
not_do_steps = options.include?(:not_do)? options[:not_do].split(",") : all_steps - do_steps
|
|
66
|
+
|
|
67
|
+
do_steps << 'base_system' if options[:base_system]
|
|
68
|
+
do_steps << 'user' if options[:workflow]
|
|
69
|
+
|
|
70
|
+
OPTIMIZE = options[:optimize]
|
|
71
|
+
USER = options[:user] || 'rbbt'
|
|
72
|
+
CONTAINER_DEP = options[:container_dependency] || 'alpine'
|
|
73
|
+
BASE_SYSTEM = options[:base_system] || CONTAINER_DEP
|
|
81
74
|
|
|
82
75
|
VARIABLES={
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
76
|
+
:RBBT_LOG => 0,
|
|
77
|
+
:BOOTSTRAP_WORKFLOWS => (options[:workflow] || "").split(/[\s,]+/)*" ",
|
|
78
|
+
:REMOTE_RESOURCES => (options[:remote_resources] || "").split(/[\s,]+/)*" "
|
|
86
79
|
}
|
|
87
80
|
|
|
88
|
-
VARIABLES[:BOOTSTRAP_WORKFLOWS] = "" if VARIABLES[:BOOTSTRAP_WORKFLOWS] == 'none'
|
|
89
|
-
|
|
90
81
|
VARIABLES[:RBBT_SERVER] = options[:server] if options[:server]
|
|
91
82
|
VARIABLES[:RBBT_FILE_SERVER] = options[:file_server] if options[:file_server]
|
|
92
83
|
VARIABLES[:RBBT_WORKFLOW_SERVER] = options[:workflow_server] if options[:workflow_server]
|
|
@@ -96,213 +87,88 @@ VARIABLES[:RBBT_NO_PROGRESS] = "true" if options[:nobar]
|
|
|
96
87
|
|
|
97
88
|
options[:ruby_version] ||= "2.6.4"
|
|
98
89
|
|
|
99
|
-
|
|
100
90
|
provision_script =<<-EOF
|
|
101
91
|
#!/bin/bash -x
|
|
102
92
|
|
|
103
|
-
|
|
104
|
-
echo
|
|
93
|
+
# PROVISION FILE
|
|
105
94
|
echo "CMD: #{File.basename($0) + " " + orig_argv.collect{|a| a =~ /\s/ ? "\'#{a}\'" : a }.join(" ")}"
|
|
106
|
-
echo
|
|
107
|
-
echo -n "Starting: "
|
|
108
|
-
date
|
|
109
|
-
|
|
110
|
-
EOF
|
|
111
|
-
|
|
112
|
-
provision_script +=<<-EOF
|
|
113
|
-
echo "1. Provisioning base system"
|
|
114
|
-
#{
|
|
115
|
-
if not SKIP_BASE_SYSTEM
|
|
116
|
-
File.read(script_dir + 'ubuntu_setup.sh')
|
|
117
|
-
else
|
|
118
|
-
"echo SKIPPED\necho"
|
|
119
|
-
end
|
|
120
|
-
}
|
|
121
|
-
|
|
122
|
-
#{
|
|
123
|
-
if not SKIP_BASE_SYSTEM and R_CUSTOM
|
|
124
|
-
"echo 1.1 Setting custom R"
|
|
125
|
-
File.read(script_dir + 'R_setup.sh')
|
|
126
|
-
end
|
|
127
|
-
}
|
|
128
|
-
|
|
129
|
-
#{
|
|
130
|
-
if not SKIP_R
|
|
131
|
-
"echo 1.2 Install R packages"
|
|
132
|
-
File.read(script_dir + 'R_packages.sh')
|
|
133
|
-
end
|
|
134
|
-
}
|
|
135
|
-
|
|
136
|
-
echo "2. Setting up tokyocabinet"
|
|
137
|
-
#{
|
|
138
|
-
if not SKIP_TOKYOCABINET
|
|
139
|
-
File.read(script_dir + 'tokyocabinet_setup.sh')
|
|
140
|
-
else
|
|
141
|
-
"echo SKIPPED\necho"
|
|
142
|
-
end
|
|
143
|
-
}
|
|
144
|
-
|
|
145
|
-
echo "3. Setting up ruby"
|
|
146
|
-
#{
|
|
147
|
-
if not SKIP_RUBY
|
|
148
|
-
"export RUBY_VERSION='#{options[:ruby_version]}'\n" << File.read(script_dir + 'ruby_setup.sh')
|
|
149
|
-
else
|
|
150
|
-
"echo SKIPPED\necho"
|
|
151
|
-
end
|
|
152
|
-
}
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
echo "3.1. Setting up gems"
|
|
156
|
-
#{
|
|
157
|
-
if not SKIP_GEM
|
|
158
|
-
File.read(script_dir + 'gem_setup.sh')
|
|
159
|
-
else
|
|
160
|
-
"echo SKIPPED\necho"
|
|
161
|
-
end
|
|
162
|
-
}
|
|
163
|
-
|
|
164
|
-
echo "4 Setting up other stuff"
|
|
165
|
-
echo "4.1 Setting up perl"
|
|
166
|
-
#{
|
|
167
|
-
if not SKIP_PERL
|
|
168
|
-
File.read(script_dir + 'perl_setup.sh')
|
|
169
|
-
else
|
|
170
|
-
"echo SKIPPED\necho"
|
|
171
|
-
end
|
|
172
|
-
}
|
|
173
|
-
|
|
174
|
-
echo "4.2 Setting up python"
|
|
175
|
-
#{
|
|
176
|
-
if not SKIP_PYTHON
|
|
177
|
-
File.read(script_dir + 'python_setup.sh')
|
|
178
|
-
else
|
|
179
|
-
"echo SKIPPED\necho"
|
|
180
|
-
end
|
|
181
|
-
}
|
|
182
|
-
|
|
183
|
-
echo "[ -f ~/.rbbt/etc/environment ] && . ~/.rbbt/etc/environment" >> "/etc/rbbt_environment"
|
|
184
|
-
echo "source /etc/rbbt_environment" >> /etc/profile
|
|
185
|
-
EOF
|
|
186
|
-
|
|
187
|
-
provision_script +=<<-EOF
|
|
188
|
-
echo "4. Configuring user"
|
|
189
|
-
EOF
|
|
190
|
-
|
|
191
|
-
if not SKIP_USER
|
|
192
|
-
provision_script +=<<-EOF
|
|
193
|
-
####################
|
|
194
|
-
# USER CONFIGURATION
|
|
195
|
-
|
|
196
|
-
if [[ '#{ USER }' == 'root' ]] ; then
|
|
197
|
-
home_dir='/root'
|
|
198
|
-
else
|
|
199
|
-
useradd -ms /bin/bash #{USER}
|
|
200
|
-
home_dir='/home/#{USER}'
|
|
201
|
-
fi
|
|
202
95
|
|
|
203
|
-
|
|
204
|
-
|
|
205
|
-
chown -R #{USER} /home/#{USER}/.rbbt/
|
|
96
|
+
test -f /etc/profile && source /etc/profile
|
|
97
|
+
test -f /etc/rbbt_environment && source /etc/rbbt_environment
|
|
206
98
|
|
|
207
|
-
|
|
208
|
-
# set user configuration script
|
|
209
|
-
cat > $user_script <<'EUSER'
|
|
210
|
-
|
|
211
|
-
. /etc/profile
|
|
212
|
-
|
|
213
|
-
echo "4.1. Loading custom variables"
|
|
214
|
-
#{
|
|
215
|
-
VARIABLES.collect do |variable,value|
|
|
216
|
-
"export " << ([variable,'"' << value.to_s << '"'] * "=")
|
|
217
|
-
end * "\n"
|
|
218
|
-
}
|
|
219
|
-
|
|
220
|
-
echo "4.2. Loading default variables"
|
|
221
|
-
#{File.read(script_dir + 'variables.sh')}
|
|
222
|
-
|
|
223
|
-
echo "4.3. Configuring rbbt"
|
|
224
|
-
#{File.read(script_dir + 'user_setup.sh')}
|
|
225
|
-
EUSER
|
|
226
|
-
|
|
227
|
-
echo "4.4. Running user configuration as '#{USER}'"
|
|
228
|
-
chown #{USER} $user_script;
|
|
229
|
-
su -l -c "bash $user_script" #{USER}
|
|
230
|
-
|
|
231
|
-
EOF
|
|
232
|
-
else
|
|
233
|
-
provision_script += "echo SKIPPED\necho\n\n"
|
|
234
|
-
end
|
|
235
|
-
|
|
236
|
-
provision_script +=<<-EOF
|
|
237
|
-
echo "5. Bootstrapping workflows as '#{USER}'"
|
|
238
|
-
echo
|
|
239
99
|
EOF
|
|
240
100
|
|
|
241
|
-
|
|
242
|
-
|
|
243
|
-
|
|
244
|
-
|
|
245
|
-
|
|
246
|
-
else
|
|
247
|
-
home_dir='/home/#{USER}'
|
|
248
|
-
fi
|
|
249
|
-
|
|
250
|
-
user_script=$home_dir/.rbbt/bin/bootstrap
|
|
251
|
-
|
|
252
|
-
cat > $user_script <<'EUSER'
|
|
253
|
-
|
|
254
|
-
. /etc/profile
|
|
255
|
-
|
|
256
|
-
echo "5.1. Loading custom variables"
|
|
257
|
-
#{
|
|
258
|
-
VARIABLES.collect do |variable,value|
|
|
259
|
-
"export " << ([variable,'"' << value.to_s << '"'] * "=")
|
|
260
|
-
end * "\n"
|
|
261
|
-
}
|
|
262
|
-
|
|
263
|
-
echo "5.2. Loading default variables"
|
|
264
|
-
#{File.read(script_dir + 'variables.sh')}
|
|
265
|
-
|
|
266
|
-
echo "5.3. Configuring rbbt"
|
|
267
|
-
#{File.read(script_dir + 'user_setup.sh')}
|
|
268
|
-
#
|
|
269
|
-
echo "5.4. Install and bootstrap"
|
|
270
|
-
#{File.read(script_dir + "bootstrap.sh")}
|
|
271
|
-
EUSER
|
|
272
|
-
|
|
273
|
-
chown #{USER} $user_script;
|
|
274
|
-
su -l -c "bash $user_script" #{USER}
|
|
101
|
+
all_steps.each_with_index do |step,i|
|
|
102
|
+
if ! do_steps.include?(step)
|
|
103
|
+
provision_script += "#" + "NOT DO #{step}\n"
|
|
104
|
+
next
|
|
105
|
+
end
|
|
275
106
|
|
|
276
|
-
|
|
277
|
-
|
|
278
|
-
|
|
107
|
+
provision_script += "#" + "DO #{step}\n"
|
|
108
|
+
provision_script += case step
|
|
109
|
+
when 'base_system'
|
|
110
|
+
File.read(script_dir + "#{BASE_SYSTEM}_setup.sh")
|
|
111
|
+
when 'user'
|
|
112
|
+
user_script =<<~EOF
|
|
113
|
+
. /etc/profile
|
|
114
|
+
. /etc/rbbt_environment
|
|
115
|
+
|
|
116
|
+
echo "6.1. Loading custom variables"
|
|
117
|
+
#{
|
|
118
|
+
VARIABLES.collect do |variable,value|
|
|
119
|
+
"export " << ([variable,'"' << value.to_s << '"'] * "=")
|
|
120
|
+
end * "\n"
|
|
121
|
+
}
|
|
122
|
+
|
|
123
|
+
echo "6.2. Loading default variables"
|
|
124
|
+
#{File.read(script_dir + 'variables.sh')}
|
|
125
|
+
|
|
126
|
+
echo "6.3. Configuring rbbt"
|
|
127
|
+
#{File.read(script_dir + 'user.sh')}
|
|
128
|
+
|
|
129
|
+
echo "6.4. Install and bootstrap"
|
|
130
|
+
#{File.read(script_dir + "bootstrap.sh")}
|
|
131
|
+
|
|
132
|
+
echo "6.5. Migrate shared files"
|
|
133
|
+
#{File.read(script_dir + "migrate.sh")}
|
|
134
|
+
EOF
|
|
135
|
+
|
|
136
|
+
<<~EOF
|
|
137
|
+
if [[ '#{ USER }' == 'root' ]] ; then
|
|
138
|
+
home_dir='/root'
|
|
139
|
+
else
|
|
140
|
+
adduser --disabled-password --gecos "" #{USER}
|
|
141
|
+
addgroup rbbt
|
|
142
|
+
adduser #{USER} rbbt
|
|
143
|
+
home_dir='/home/#{USER}'
|
|
144
|
+
chown -R #{USER} $home_dir/
|
|
145
|
+
fi
|
|
146
|
+
|
|
147
|
+
mkdir -p $home_dir/.rbbt/bin/
|
|
148
|
+
user_script=$home_dir/.rbbt/bin/bootstrap
|
|
149
|
+
chown -R #{USER} $home_dir/
|
|
150
|
+
|
|
151
|
+
for d in /usr/local/var/rbbt /usr/local/share/rbbt /usr/local/workflows/rbbt /usr/local/software/rbbt; do
|
|
152
|
+
mkdir -p $d
|
|
153
|
+
chgrp rbbt $d
|
|
154
|
+
chown rbbt $d
|
|
155
|
+
chmod g+w $d
|
|
156
|
+
done
|
|
157
|
+
|
|
158
|
+
cat > $user_script <<'EUSER'
|
|
159
|
+
#{user_script}
|
|
160
|
+
EUSER
|
|
161
|
+
|
|
162
|
+
|
|
163
|
+
su -l -c "sh $user_script" #{USER}
|
|
164
|
+
EOF
|
|
165
|
+
else
|
|
166
|
+
File.read(script_dir + "#{step}.sh")
|
|
167
|
+
end
|
|
279
168
|
end
|
|
280
169
|
|
|
281
|
-
|
|
282
|
-
|
|
283
|
-
# =====
|
|
284
|
-
|
|
285
|
-
#{File.read(script_dir + 'hacks.sh')}
|
|
286
|
-
EOF
|
|
287
|
-
|
|
288
|
-
provision_script +=<<-EOF
|
|
289
|
-
# CODA
|
|
290
|
-
# ====
|
|
291
|
-
|
|
292
|
-
apt-get clean
|
|
293
|
-
rm -rf /var/lib/apt/lists/* /var/tmp/* /usr/share/doc /usr/share/man /usr/local/share/ri
|
|
294
|
-
|
|
295
|
-
#{ "su -l -c 'rbbt system optimize /home/#{USER}/.rbbt ' #{USER}" if OPTIMIZE}
|
|
296
|
-
|
|
297
|
-
echo
|
|
298
|
-
echo -n "Installation done: "
|
|
299
|
-
date
|
|
300
|
-
|
|
301
|
-
EOF
|
|
302
|
-
|
|
303
|
-
|
|
304
|
-
if docker_image = options[:docker]
|
|
305
|
-
container_dependency = options[:container_dependency] || 'ubuntu:focal'
|
|
170
|
+
if docker_image = options[:docker]
|
|
171
|
+
container_dependency = CONTAINER_DEP
|
|
306
172
|
dockerfile = options[:dockerfile] || File.join(root_dir, 'Dockerfile')
|
|
307
173
|
dockerfile_text = Open.read(dockerfile)
|
|
308
174
|
dockerfile_text.sub!(/^FROM.*/,'FROM ' + container_dependency) if container_dependency
|
|
@@ -339,31 +205,44 @@ end
|
|
|
339
205
|
|
|
340
206
|
if singularity_image = options[:singularity]
|
|
341
207
|
|
|
342
|
-
container_dependency =
|
|
208
|
+
container_dependency = CONTAINER_DEP
|
|
343
209
|
TmpFile.with_file(nil, false) do |dir|
|
|
344
210
|
Path.setup(dir)
|
|
345
211
|
|
|
346
212
|
provision_file = dir['provision.sh']
|
|
347
213
|
|
|
348
|
-
|
|
214
|
+
if container_dependency.include? '.sif'
|
|
215
|
+
bootstrap_text=<<-EOF
|
|
216
|
+
Bootstrap: localimage
|
|
217
|
+
From: #{container_dependency}
|
|
218
|
+
EOF
|
|
219
|
+
else
|
|
220
|
+
bootstrap_text=<<-EOF
|
|
349
221
|
Bootstrap: docker
|
|
350
222
|
From: #{container_dependency}
|
|
223
|
+
EOF
|
|
224
|
+
end
|
|
351
225
|
|
|
226
|
+
bootstrap_text+=<<-EOF
|
|
352
227
|
%post
|
|
353
228
|
cat > /image_provision.sh <<"EOS"
|
|
354
229
|
#{provision_script}
|
|
355
230
|
EOS
|
|
356
|
-
|
|
357
|
-
ln -s /etc/rbbt_environment /.singularity.d/env/99-rbbt_environment.sh
|
|
231
|
+
sh -x /image_provision.sh 2>&1 | tee /image_provision.log
|
|
232
|
+
bash -c '[ -f /.singularity.d/env/99-rbbt_environment.sh ] || ln -s /etc/rbbt_environment /.singularity.d/env/99-rbbt_environment.sh'
|
|
358
233
|
chmod +x /.singularity.d/env/99-rbbt_environment.sh
|
|
359
|
-
bash -c '[ -d /
|
|
360
|
-
bash -c '[ -d /
|
|
361
|
-
bash -c '[ -d /
|
|
362
|
-
bash -c '[ -d /
|
|
363
|
-
bash -c '[ -d /
|
|
364
|
-
bash -c '[ -d /home/#{USER}/.rbbt/
|
|
365
|
-
bash -c '[ -d /home/#{USER}/.rbbt/
|
|
366
|
-
bash -c '[ -d /home/#{USER}/.rbbt/
|
|
234
|
+
#bash -c '[ -d /home/#{USER}/.local/lib/ ] && (rsync -av /home/#{USER}/.local/lib/ /usr/lib/' && chmod 755 /usr/lib/python*/site-packages/ && chown -R root /usr/lib/python*/site-packages/ && rm /home/#{USER}/.local/lib/) || echo -n ""
|
|
235
|
+
#bash -c '[ -d /usr/local/share ] || mkdir -p /usr/local/share'
|
|
236
|
+
#bash -c '[ -d /usr/local/share ] || mkdir -p /usr/local/share'
|
|
237
|
+
#bash -c '[ -d /usr/local/workflows ] || mkdir -p /usr/local/workflows'
|
|
238
|
+
#bash -c '[ -d /software/rbbt ] || mkdir -p /software/rbbt'
|
|
239
|
+
#bash -c '[ -d /home/#{USER}/.rbbt/var/ ] && mv /home/#{USER}/.rbbt/var/ /var/rbbt' || echo -n ""
|
|
240
|
+
#bash -c '[ -d /home/#{USER}/.rbbt/var/ ] && rm -Rf /home/#{USER}/.rbbt/var/' || echo -n ""
|
|
241
|
+
#bash -c '[ -d /home/#{USER}/.rbbt/share/ ] && mv /home/#{USER}/.rbbt/share/ /usr/local/share/rbbt' || echo -n ""
|
|
242
|
+
#bash -c '[ -d /home/#{USER}/.rbbt/workflows/ ] && mv /home/#{USER}/.rbbt/workflows/ /usr/local/workflows/rbbt' || echo -n ""
|
|
243
|
+
#bash -c '[ -d /home/#{USER}/.rbbt/software/opt ] && mv /home/#{USER}/.rbbt/software/opt /software/rbbt/opt' || echo -n ""
|
|
244
|
+
#bash -c '[ -d /home/#{USER}/.rbbt/software/src ] && mv /home/#{USER}/.rbbt/software/src /software/rbbt/src' || echo -n ""
|
|
245
|
+
#bash -c '[ -d /home/#{USER}/.rbbt/software/scm ] && mv /home/#{USER}/.rbbt/software/scm /software/rbbt/scm' || echo -n ""
|
|
367
246
|
EOF
|
|
368
247
|
FileUtils.mkdir_p dir
|
|
369
248
|
Open.write(dir["singularity_bootstrap"].find, bootstrap_text)
|
|
@@ -374,11 +253,11 @@ EOF
|
|
|
374
253
|
cmd_boot = "singularity build #{singularity_image} '#{dir["singularity_bootstrap"]}'"
|
|
375
254
|
|
|
376
255
|
|
|
377
|
-
puts "**************"
|
|
378
|
-
puts "CREATING IMAGE: #{dir["singularity_bootstrap"].find}"
|
|
379
|
-
puts "**************"
|
|
380
|
-
puts cmd_create
|
|
381
|
-
CMD.cmd_log(cmd_create, :log => 4)
|
|
256
|
+
#puts "**************"
|
|
257
|
+
#puts "CREATING IMAGE: #{dir["singularity_bootstrap"].find}"
|
|
258
|
+
#puts "**************"
|
|
259
|
+
#puts cmd_create
|
|
260
|
+
#CMD.cmd_log(cmd_create, :log => 4)
|
|
382
261
|
|
|
383
262
|
puts "**************"
|
|
384
263
|
puts "BUILDING IMAGE: #{dir["singularity_bootstrap"].find}"
|
|
@@ -390,6 +269,7 @@ EOF
|
|
|
390
269
|
end
|
|
391
270
|
|
|
392
271
|
if options[:virtualbox]
|
|
272
|
+
container_dependency = CONTAINER_DEP
|
|
393
273
|
TmpFile.with_file(nil, false) do |dir|
|
|
394
274
|
Path.setup(dir)
|
|
395
275
|
|
|
@@ -416,11 +296,9 @@ Vagrant.configure(VAGRANTFILE_API_VERSION) do |config|
|
|
|
416
296
|
end
|
|
417
297
|
EOF
|
|
418
298
|
|
|
299
|
+
Log.debug "Running vagrant on #{dir}"
|
|
419
300
|
Misc.in_dir(dir) do
|
|
420
|
-
|
|
421
|
-
while line = io.gets
|
|
422
|
-
puts line
|
|
423
|
-
end
|
|
301
|
+
CMD.cmd_log(:vagrant, "up")
|
|
424
302
|
end
|
|
425
303
|
end
|
|
426
304
|
end
|
|
@@ -0,0 +1,66 @@
|
|
|
1
|
+
#!/bin/bash -x
|
|
2
|
+
|
|
3
|
+
# INSTALL
|
|
4
|
+
# =======
|
|
5
|
+
|
|
6
|
+
# R packages
|
|
7
|
+
# ----------
|
|
8
|
+
|
|
9
|
+
function R_install_packages(){
|
|
10
|
+
pkgs="'$1'"
|
|
11
|
+
shift
|
|
12
|
+
for p in $@; do
|
|
13
|
+
pkgs="$pkgs, '$p'"
|
|
14
|
+
done
|
|
15
|
+
echo "install.packages(c($pkgs), repos='http://cran.us.r-project.org')" | R --vanilla --quiet
|
|
16
|
+
}
|
|
17
|
+
|
|
18
|
+
function R_biocLite(){
|
|
19
|
+
pkgs="'$1'"
|
|
20
|
+
shift
|
|
21
|
+
for p in $@; do
|
|
22
|
+
echo "BiocManager::install(c('$p'))" | R --vanilla --quiet
|
|
23
|
+
done
|
|
24
|
+
}
|
|
25
|
+
|
|
26
|
+
function R_CMD_install(){
|
|
27
|
+
url=$1
|
|
28
|
+
name=$2
|
|
29
|
+
wget "$url" -O /tmp/R-pkg-"$name".tar.gz
|
|
30
|
+
R CMD INSTALL /tmp/R-pkg-"$name".tar.gz
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
export R_HOME=$(R RHOME)
|
|
35
|
+
export R_INCLUDE="$R_HOME/../../include/R"
|
|
36
|
+
gem install rsruby -- --with-R-dir=$R_HOME --with-R-include=$R_INCLUDE
|
|
37
|
+
|
|
38
|
+
echo 'install.packages("Rserve", repos="http://www.rforge.net/")' | R --vanilla --quiet
|
|
39
|
+
gem install rserve-client
|
|
40
|
+
|
|
41
|
+
#R_install_packages 'Rcpp'
|
|
42
|
+
#R_CMD_install 'https://cran.r-project.org/src/contrib/Archive/plyr/plyr_1.8.1.tar.gz' plyr
|
|
43
|
+
|
|
44
|
+
#R_install_packages 'MASS' 'nnet'
|
|
45
|
+
#R_CMD_install 'https://cran.r-project.org/src/contrib/Archive/car/car_2.0-22.tar.gz' car
|
|
46
|
+
|
|
47
|
+
R_install_packages 'tidyverse'
|
|
48
|
+
|
|
49
|
+
R_install_packages 'RJSONIO' 'XML' 'digest' 'gtable' 'reshape2' 'scales' 'proto'
|
|
50
|
+
#R_CMD_install 'https://cran.r-project.org/src/contrib/Archive/ggplot2/ggplot2_1.0.0.tar.gz' ggplot2
|
|
51
|
+
|
|
52
|
+
R_install_packages 'BiocManager'
|
|
53
|
+
R_biocLite Biobase
|
|
54
|
+
|
|
55
|
+
R_install_packages NMF Cairo drc gridSVG ggthemes mclust randomForest viper
|
|
56
|
+
|
|
57
|
+
R_install_packages pheatmap VennDiagram Hmisc pls gridSVG
|
|
58
|
+
|
|
59
|
+
R_install_packages UpSetR
|
|
60
|
+
|
|
61
|
+
R_biocLite limma viper
|
|
62
|
+
|
|
63
|
+
R_install_packages pROC txtplot
|
|
64
|
+
|
|
65
|
+
rm /tmp/R-pkg-*.tar.gz
|
|
66
|
+
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
# Basic alpine setup
|
|
2
|
+
apk add ruby ruby-dev # Ruby
|
|
3
|
+
apk add git make gcc g++ cmake # Building
|
|
4
|
+
apk add bzip2 bzip2-dev zlib zlib-dev krb5 gcompat # Libs
|
|
5
|
+
apk add bash openssh-client wget curl rsync gnu-libiconv # Tools
|
|
6
|
+
|
|
7
|
+
gem install rbbt-util rbbt-sources
|
|
8
|
+
gem install RubyInline
|
|
9
|
+
|
|
10
|
+
gem install specific_install hoe minitest
|
|
11
|
+
gem specific_install -l https://github.com/mikisvaz/lockfile.git
|
|
12
|
+
gem specific_install -l https://github.com/mikisvaz/rubyinline.git
|
|
13
|
+
|
|
14
|
+
rm -Rf /usr/lib/ruby/gems/*/doc /usr/lib/ruby/gems/*/cache
|
|
15
|
+
|
|
16
|
+
grep rbbt_environment /etc/rbbt_environment || echo "[ -f ~/.rbbt/etc/environment ] && . ~/.rbbt/etc/environment" >> "/etc/rbbt_environment"
|
|
17
|
+
grep rbbt_environment /etc/profile || echo echo "source /etc/rbbt_environment" >> /etc/profile
|
|
@@ -0,0 +1,47 @@
|
|
|
1
|
+
# Basic alpine setup
|
|
2
|
+
apk add ruby ruby-dev # Ruby
|
|
3
|
+
apk add git make gcc g++ cmake # Building
|
|
4
|
+
apk add bzip2 bzip2-dev zlib zlib-dev krb5 gcompat # Libs
|
|
5
|
+
apk add bash openssh-client wget curl rsync gnu-libiconv # Tools
|
|
6
|
+
|
|
7
|
+
gem install rbbt-util rbbt-sources
|
|
8
|
+
gem install RubyInline
|
|
9
|
+
|
|
10
|
+
gem install specific_install hoe minitest
|
|
11
|
+
gem specific_install -l https://github.com/mikisvaz/lockfile.git
|
|
12
|
+
gem specific_install -l https://github.com/mikisvaz/rubyinline.git
|
|
13
|
+
|
|
14
|
+
rm -Rf /usr/lib/ruby/gems/*/doc /usr/lib/ruby/gems/*/cache
|
|
15
|
+
|
|
16
|
+
grep rbbt_environment /etc/rbbt_environment || echo "[ -f ~/.rbbt/etc/environment ] && . ~/.rbbt/etc/environment" >> "/etc/rbbt_environment"
|
|
17
|
+
grep rbbt_environment /etc/profile || echo echo "source /etc/rbbt_environment" >> /etc/profile
|
|
18
|
+
|
|
19
|
+
# Main alpine setup
|
|
20
|
+
apk add xvfb bison autoconf rsync curl openssl numactl zlib-dev zlib yaml-dev openssl xz-dev xz-libs libffi tcsh gawk pandoc libtbb-dev yaml libxml2-dev libxml2 shared-mime-info
|
|
21
|
+
|
|
22
|
+
# Extended alpine setup
|
|
23
|
+
apk add openjdk17 R R-dev python3 python3-dev py3-pip ansible
|
|
24
|
+
|
|
25
|
+
apk add fontconfig fontconfig-dev harfbuzz fribidi harfbuzz-dev fribidi-dev jpeg jpeg-dev tiff tiff-dev cairo cairo-dev libxt-dev libxt
|
|
26
|
+
|
|
27
|
+
# Allow USER to install things globaly during bootstrap
|
|
28
|
+
chmod 777 -R /usr/lib/python*/site-packages
|
|
29
|
+
|
|
30
|
+
gem install pycall
|
|
31
|
+
|
|
32
|
+
export R_HOME=$(R RHOME)
|
|
33
|
+
export R_INCLUDE="$R_HOME/../../include/R"
|
|
34
|
+
cat >> /etc/rbbt_environment <<'EOF'
|
|
35
|
+
export R_HOME=$(R RHOME)
|
|
36
|
+
export R_INCLUDE="$R_HOME/../../include/R"
|
|
37
|
+
export LD_LIBRARY_PATH="$LD_LIBRARY_PATH:$R_HOME/lib"
|
|
38
|
+
export PATH="$PATH:$R_HOME/bin"
|
|
39
|
+
EOF
|
|
40
|
+
gem install rsruby -- --with-R-dir=$R_HOME --with-R-include=$R_INCLUDE
|
|
41
|
+
|
|
42
|
+
echo 'install.packages("Rserve", repos="http://www.rforge.net/")' | R --vanilla --quiet
|
|
43
|
+
gem install rserve-client
|
|
44
|
+
|
|
45
|
+
export JAVA_HOME=$(java -XshowSettings:properties -version 2>&1 > /dev/null | grep 'java.home' | cut -f 2 -d "=" | awk '{$1=$1};1')
|
|
46
|
+
echo $JAVA_HOME
|
|
47
|
+
gem install rjb
|
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
# Basic alpine setup
|
|
2
|
+
apk add ruby ruby-dev # Ruby
|
|
3
|
+
apk add git make gcc g++ cmake # Building
|
|
4
|
+
apk add bzip2 bzip2-dev zlib zlib-dev krb5 gcompat # Libs
|
|
5
|
+
apk add bash openssh-client wget curl rsync gnu-libiconv # Tools
|
|
6
|
+
|
|
7
|
+
gem install rbbt-util rbbt-sources
|
|
8
|
+
gem install RubyInline
|
|
9
|
+
|
|
10
|
+
gem install specific_install hoe minitest
|
|
11
|
+
gem specific_install -l https://github.com/mikisvaz/lockfile.git
|
|
12
|
+
gem specific_install -l https://github.com/mikisvaz/rubyinline.git
|
|
13
|
+
|
|
14
|
+
rm -Rf /usr/lib/ruby/gems/*/doc /usr/lib/ruby/gems/*/cache
|
|
15
|
+
|
|
16
|
+
grep rbbt_environment /etc/rbbt_environment || echo "[ -f ~/.rbbt/etc/environment ] && . ~/.rbbt/etc/environment" >> "/etc/rbbt_environment"
|
|
17
|
+
grep rbbt_environment /etc/profile || echo echo "source /etc/rbbt_environment" >> /etc/profile
|
|
18
|
+
|
|
19
|
+
# Main alpine setup
|
|
20
|
+
apk add xvfb bison autoconf rsync curl openssl numactl zlib-dev zlib yaml-dev openssl xz-dev xz-libs libffi tcsh gawk pandoc libtbb-dev yaml libxml2-dev libxml2 shared-mime-info
|
|
@@ -3,15 +3,27 @@
|
|
|
3
3
|
# USER RBBT BOOTSTRAP
|
|
4
4
|
# ===================
|
|
5
5
|
|
|
6
|
+
export RBBT_WORKFLOW_AUTOINSTALL=true
|
|
7
|
+
export RBBT_LOG=0
|
|
8
|
+
|
|
6
9
|
for workflow in $BOOTSTRAP_WORKFLOWS; do
|
|
7
10
|
rbbt workflow install $workflow
|
|
8
|
-
done
|
|
9
11
|
|
|
10
|
-
|
|
11
|
-
|
|
12
|
+
pip_requirements_file=$(rbbt_find.rb -w $workflow requirements.pip --nocolor)
|
|
13
|
+
test -z $pip_requirements_file || pip install -r $pip_requirements_file
|
|
14
|
+
unset pip_requirements_file
|
|
15
|
+
|
|
16
|
+
pip_requirements_file2=$(rbbt_find.rb -w $workflow requirements.pip2 --nocolor)
|
|
17
|
+
test -z $pip_requirements_file2 || pip install -r $pip_requirements_file2
|
|
18
|
+
unset pip_requirements_file2
|
|
19
|
+
|
|
20
|
+
pip_requirements_file3=$(rbbt_find.rb -w $workflow requirements.pip3 --nocolor)
|
|
21
|
+
test -z $pip_requirements_file3 || pip install -r $pip_requirements_file3
|
|
22
|
+
unset pip_requirements_file3
|
|
23
|
+
done
|
|
12
24
|
|
|
13
25
|
for workflow in $BOOTSTRAP_WORKFLOWS; do
|
|
14
26
|
echo "Bootstrapping $workflow on $BOOTSTRAP_CPUS CPUs"
|
|
15
27
|
[ -d $HOME/.rbbt/tmp/ ] || mkdir -p $HOME/.rbbt/tmp/
|
|
16
|
-
rbbt workflow cmd $workflow bootstrap --cpus $BOOTSTRAP_CPUS
|
|
28
|
+
rbbt workflow cmd $workflow bootstrap --config_keys "cpus $BOOTSTRAP_CPUS bootstrap" > $HOME/.rbbt/tmp/${workflow}.bootstrap.log
|
|
17
29
|
done
|
|
@@ -0,0 +1,58 @@
|
|
|
1
|
+
_var_content(){
|
|
2
|
+
var=$1
|
|
3
|
+
eval echo "$"$(echo $var)
|
|
4
|
+
}
|
|
5
|
+
|
|
6
|
+
_append_with_colon(){
|
|
7
|
+
var=$1
|
|
8
|
+
value=$2
|
|
9
|
+
|
|
10
|
+
current=`_var_content $var`
|
|
11
|
+
|
|
12
|
+
if ! echo $current | grep "\(^\|:\)$value\(:\|$\)" >/dev/null 2>&1; then
|
|
13
|
+
eval $(echo $var)="\"$current:$value\""
|
|
14
|
+
fi
|
|
15
|
+
}
|
|
16
|
+
|
|
17
|
+
_append_with_space(){
|
|
18
|
+
var=$1
|
|
19
|
+
value=$2
|
|
20
|
+
|
|
21
|
+
current=`_var_content $var`
|
|
22
|
+
|
|
23
|
+
if ! echo $current | grep "\(^\| \)$value\( \|$\)" >/dev/null 2>&1; then
|
|
24
|
+
eval $(echo $var)="\"$current:$value\""
|
|
25
|
+
fi
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
_prepend_with_colon(){
|
|
29
|
+
var=$1
|
|
30
|
+
value=$2
|
|
31
|
+
|
|
32
|
+
current=`_var_content $var`
|
|
33
|
+
|
|
34
|
+
if ! echo $current | grep "\(^\|:\)$value\(:\|$\)" >/dev/null 2>&1; then
|
|
35
|
+
eval $(echo $var)="\"$value:$current\""
|
|
36
|
+
fi
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
_prepend_with_space(){
|
|
40
|
+
var=$1
|
|
41
|
+
value=$2
|
|
42
|
+
|
|
43
|
+
current=`_var_content $var`
|
|
44
|
+
|
|
45
|
+
if ! echo $current | grep "\(^\| \)$value\( \|$\)" >/dev/null 2>&1; then
|
|
46
|
+
eval $(echo $var)="\"$value $current\""
|
|
47
|
+
fi
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
_add_path(){
|
|
51
|
+
_prepend_with_colon PATH "${1%/}"
|
|
52
|
+
}
|
|
53
|
+
|
|
54
|
+
_add_libpath(){
|
|
55
|
+
_prepend_with_colon LD_LIBRARY_PATH "${1%/}"
|
|
56
|
+
_prepend_with_colon LD_RUN_PATH "${1%/}"
|
|
57
|
+
}
|
|
58
|
+
|
|
@@ -0,0 +1,42 @@
|
|
|
1
|
+
#!/bin/bash -x
|
|
2
|
+
|
|
3
|
+
# RUBY GEMS and RBBT
|
|
4
|
+
# =================
|
|
5
|
+
|
|
6
|
+
echo "* Updating RubyGem"
|
|
7
|
+
export REALLY_GEM_UPDATE_SYSTEM=true
|
|
8
|
+
env REALLY_GEM_UPDATE_SYSTEM=true gem update --no-document --system
|
|
9
|
+
|
|
10
|
+
#echo "* Installing difficult gems: ZenTest, RubyInline, rsruby and rjb"
|
|
11
|
+
#gem install --force --no-document RubyInline
|
|
12
|
+
gem install --force --no-document ZenTest
|
|
13
|
+
|
|
14
|
+
## R (extra config in gem)
|
|
15
|
+
#export R_INCLUDE="$(echo "$R_HOME" | sed 's@/usr/lib\(32\|64\)*@/usr/share@')/include"
|
|
16
|
+
#gem install --conservative --no-document rsruby -- --with-R-dir="$R_HOME" --with-R-include="$R_INCLUDE" \
|
|
17
|
+
# --with_cflags="-fPIC -g -O2 -fstack-protector --param=ssp-buffer-size=4 -Wformat -Wall -fno-strict-aliasing"
|
|
18
|
+
|
|
19
|
+
# Java (extra config in gem)
|
|
20
|
+
#export JAVA_HOME=$(echo /usr/lib/jvm/java-?-openjdk-*)
|
|
21
|
+
#gem install --conservative --force --no-document rjb
|
|
22
|
+
|
|
23
|
+
echo "* Installing all rbbt gems"
|
|
24
|
+
# Rbbt and some optional gems
|
|
25
|
+
gem install --no-document --force \
|
|
26
|
+
tokyocabinet \
|
|
27
|
+
ruby-prof \
|
|
28
|
+
rbbt-util rbbt-sources rbbt-dm rbbt-text rbbt-rest
|
|
29
|
+
|
|
30
|
+
echo "* Installing extra gems for web stuff"
|
|
31
|
+
# Extra things for web interface
|
|
32
|
+
gem install --no-document \
|
|
33
|
+
sinatra puma \
|
|
34
|
+
rest-client \
|
|
35
|
+
kramdown pandoc pandoc-ruby \
|
|
36
|
+
spreadsheet rubyXL \
|
|
37
|
+
prawn prawn-svg mimemagic \
|
|
38
|
+
prime mechanize
|
|
39
|
+
|
|
40
|
+
gem install --no-document mini_racer uglifier
|
|
41
|
+
|
|
42
|
+
gem cleanup
|
|
@@ -1,2 +1,2 @@
|
|
|
1
1
|
# For BNReduction work
|
|
2
|
-
ln -s /bin/sed /usr/bin/sed
|
|
2
|
+
#ln -s /bin/sed /usr/bin/sed
|
|
@@ -2,15 +2,21 @@ wget "https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh" -O
|
|
|
2
2
|
echo "# Python conda " >> /etc/rbbt_environment
|
|
3
3
|
|
|
4
4
|
bash /tmp/miniconda.sh -b -p /usr/local/miniconda3
|
|
5
|
+
|
|
5
6
|
ln -s /usr/local/miniconda3/bin/conda /usr/local/bin/conda
|
|
6
7
|
echo '_add_path "/usr/local/miniconda3/bin/"' >> /etc/rbbt_environment
|
|
7
8
|
|
|
8
9
|
conda create --yes -n python2 python=2.7 pip
|
|
9
10
|
|
|
10
|
-
conda create --yes -n python3 python=3 pip
|
|
11
|
+
conda create --yes -n python3 python=3.6 pip
|
|
11
12
|
|
|
12
13
|
echo '_add_path "/usr/local/miniconda3/envs/python2/bin/"' >> /etc/rbbt_environment
|
|
13
14
|
echo '_add_path "/usr/local/miniconda3/envs/python3/bin/"' >> /etc/rbbt_environment
|
|
15
|
+
echo '' >> /etc/rbbt_environment
|
|
16
|
+
echo '# Setup conda environment' >> /etc/rbbt_environment
|
|
17
|
+
echo 'export CONDA_PKGS_DIRS=~/tmp/conda/pkgs' >> /etc/rbbt_environment
|
|
18
|
+
echo '. /usr/local/miniconda3/etc/profile.d/conda.sh' >> /etc/rbbt_environment
|
|
14
19
|
|
|
15
20
|
rm -Rf /usr/local/miniconda3/pkgs
|
|
16
21
|
source /etc/rbbt_environment
|
|
22
|
+
|
|
@@ -13,7 +13,7 @@ cd /tmp/ruby-install
|
|
|
13
13
|
wget https://cache.ruby-lang.org/pub/ruby/$_small_version/ruby-${RUBY_VERSION}.tar.gz -O "ruby.tar.gz"
|
|
14
14
|
tar -xvzf ruby.tar.gz
|
|
15
15
|
cd ruby-*/
|
|
16
|
-
./configure --prefix=/usr/local
|
|
16
|
+
./configure --prefix=/usr/local --enable-shared
|
|
17
17
|
make && make install
|
|
18
18
|
|
|
19
19
|
unset _small_version
|
|
@@ -0,0 +1,45 @@
|
|
|
1
|
+
#!/bin/bash -x
|
|
2
|
+
|
|
3
|
+
# Prepare SLURM loopback
|
|
4
|
+
# ======================
|
|
5
|
+
|
|
6
|
+
cat <<'EOF' > /usr/local/bin/loopback_cmd
|
|
7
|
+
#!/bin/bash
|
|
8
|
+
|
|
9
|
+
if [ -z "$LOOPBACK_KEY" ]; then
|
|
10
|
+
LOOPBACK_KEY="$HOME/.ssh/slurm-loopback"
|
|
11
|
+
fi
|
|
12
|
+
|
|
13
|
+
if [ ! -f "$LOOPBACK_KEY" ]; then
|
|
14
|
+
echo "Prepare a ssh key on '$LOOPBACK_KEY' or point to it with environment variable LOOPBACK_KEY." >&2
|
|
15
|
+
exit -1
|
|
16
|
+
else
|
|
17
|
+
ssh localhost -i $LOOPBACK_KEY "$@"
|
|
18
|
+
fi
|
|
19
|
+
EOF
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
cat <<'EOF' > /usr/local/bin/squeue
|
|
23
|
+
#!/bin/bash
|
|
24
|
+
|
|
25
|
+
loopback_cmd squeue "$@"
|
|
26
|
+
EOF
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
cat <<'EOF' > /usr/local/bin/sbatch
|
|
30
|
+
#!/bin/bash
|
|
31
|
+
|
|
32
|
+
loopback_cmd sbatch "$@"
|
|
33
|
+
EOF
|
|
34
|
+
|
|
35
|
+
cat <<'EOF' > /usr/local/bin/sacct
|
|
36
|
+
#!/bin/bash
|
|
37
|
+
|
|
38
|
+
loopback_cmd sacct "$@"
|
|
39
|
+
EOF
|
|
40
|
+
|
|
41
|
+
chmod +x /usr/local/bin/loopback_cmd /usr/local/bin/squeue /usr/local/bin/sbatch /usr/local/bin/sacct
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
|
|
@@ -3,9 +3,14 @@
|
|
|
3
3
|
# TOKYOCABINET INSTALL
|
|
4
4
|
# ===================
|
|
5
5
|
|
|
6
|
-
|
|
6
|
+
mkdir -p /opt/src/
|
|
7
|
+
cd /opt/src/
|
|
7
8
|
wget http://fallabs.com/tokyocabinet/tokyocabinet-1.4.48.tar.gz -O "tokyocabinet.tar.gz"
|
|
8
9
|
tar -xvzf tokyocabinet.tar.gz
|
|
9
10
|
cd tokyocabinet-1.4.48
|
|
10
11
|
./configure --prefix=/usr/local
|
|
11
|
-
make
|
|
12
|
+
make > /tc_make.log
|
|
13
|
+
make install > /tc_make_install.log
|
|
14
|
+
gem install tokyocabinet
|
|
15
|
+
cd /
|
|
16
|
+
rm -Rf /opt/src/
|
|
@@ -3,67 +3,65 @@
|
|
|
3
3
|
# INSTALL
|
|
4
4
|
# =======
|
|
5
5
|
|
|
6
|
-
cat << 'EOF' > /etc/rbbt_environment
|
|
7
|
-
_var_content(){
|
|
8
|
-
var=$1
|
|
9
|
-
eval echo "$"$(echo $var)
|
|
10
|
-
}
|
|
11
|
-
|
|
12
|
-
_append_with_colon(){
|
|
13
|
-
var=$1
|
|
14
|
-
value=$2
|
|
15
|
-
|
|
16
|
-
current=`_var_content $var`
|
|
17
|
-
|
|
18
|
-
if ! echo $current | grep "\(^\|:\)$value\(:\|$\)" >/dev/null 2>&1; then
|
|
19
|
-
eval $(echo $var)="\"$current:$value\""
|
|
20
|
-
fi
|
|
21
|
-
}
|
|
22
|
-
|
|
23
|
-
_append_with_space(){
|
|
24
|
-
var=$1
|
|
25
|
-
value=$2
|
|
26
|
-
|
|
27
|
-
current=`_var_content $var`
|
|
28
|
-
|
|
29
|
-
if ! echo $current | grep "\(^\| \)$value\( \|$\)" >/dev/null 2>&1; then
|
|
30
|
-
eval $(echo $var)="\"$current:$value\""
|
|
31
|
-
fi
|
|
32
|
-
}
|
|
33
|
-
|
|
34
|
-
_prepend_with_colon(){
|
|
35
|
-
var=$1
|
|
36
|
-
value=$2
|
|
37
|
-
|
|
38
|
-
current=`_var_content $var`
|
|
39
|
-
|
|
40
|
-
if ! echo $current | grep "\(^\|:\)$value\(:\|$\)" >/dev/null 2>&1; then
|
|
41
|
-
eval $(echo $var)="\"$value:$current\""
|
|
42
|
-
fi
|
|
43
|
-
}
|
|
44
|
-
|
|
45
|
-
_prepend_with_space(){
|
|
46
|
-
var=$1
|
|
47
|
-
value=$2
|
|
48
|
-
|
|
49
|
-
current=`_var_content $var`
|
|
50
|
-
|
|
51
|
-
if ! echo $current | grep "\(^\| \)$value\( \|$\)" >/dev/null 2>&1; then
|
|
52
|
-
eval $(echo $var)="\"$value $current\""
|
|
53
|
-
fi
|
|
54
|
-
}
|
|
55
|
-
|
|
56
|
-
_add_path(){
|
|
57
|
-
_prepend_with_colon PATH "${1%/}"
|
|
58
|
-
}
|
|
59
|
-
|
|
60
|
-
_add_libpath(){
|
|
61
|
-
_prepend_with_colon LD_LIBRARY_PATH "${1%/}"
|
|
62
|
-
_prepend_with_colon LD_RUN_PATH "${1%/}"
|
|
63
|
-
}
|
|
64
|
-
EOF
|
|
65
|
-
|
|
66
|
-
. /etc/rbbt_environment
|
|
6
|
+
#cat << 'EOF' > /etc/rbbt_environment
|
|
7
|
+
#_var_content(){
|
|
8
|
+
# var=$1
|
|
9
|
+
# eval echo "$"$(echo $var)
|
|
10
|
+
#}
|
|
11
|
+
#
|
|
12
|
+
#_append_with_colon(){
|
|
13
|
+
# var=$1
|
|
14
|
+
# value=$2
|
|
15
|
+
#
|
|
16
|
+
# current=`_var_content $var`
|
|
17
|
+
#
|
|
18
|
+
# if ! echo $current | grep "\(^\|:\)$value\(:\|$\)" >/dev/null 2>&1; then
|
|
19
|
+
# eval $(echo $var)="\"$current:$value\""
|
|
20
|
+
# fi
|
|
21
|
+
#}
|
|
22
|
+
#
|
|
23
|
+
#_append_with_space(){
|
|
24
|
+
# var=$1
|
|
25
|
+
# value=$2
|
|
26
|
+
#
|
|
27
|
+
# current=`_var_content $var`
|
|
28
|
+
#
|
|
29
|
+
# if ! echo $current | grep "\(^\| \)$value\( \|$\)" >/dev/null 2>&1; then
|
|
30
|
+
# eval $(echo $var)="\"$current:$value\""
|
|
31
|
+
# fi
|
|
32
|
+
#}
|
|
33
|
+
#
|
|
34
|
+
#_prepend_with_colon(){
|
|
35
|
+
# var=$1
|
|
36
|
+
# value=$2
|
|
37
|
+
#
|
|
38
|
+
# current=`_var_content $var`
|
|
39
|
+
#
|
|
40
|
+
# if ! echo $current | grep "\(^\|:\)$value\(:\|$\)" >/dev/null 2>&1; then
|
|
41
|
+
# eval $(echo $var)="\"$value:$current\""
|
|
42
|
+
# fi
|
|
43
|
+
#}
|
|
44
|
+
#
|
|
45
|
+
#_prepend_with_space(){
|
|
46
|
+
# var=$1
|
|
47
|
+
# value=$2
|
|
48
|
+
#
|
|
49
|
+
# current=`_var_content $var`
|
|
50
|
+
#
|
|
51
|
+
# if ! echo $current | grep "\(^\| \)$value\( \|$\)" >/dev/null 2>&1; then
|
|
52
|
+
# eval $(echo $var)="\"$value $current\""
|
|
53
|
+
# fi
|
|
54
|
+
#}
|
|
55
|
+
#
|
|
56
|
+
#_add_path(){
|
|
57
|
+
# _prepend_with_colon PATH "${1%/}"
|
|
58
|
+
#}
|
|
59
|
+
#
|
|
60
|
+
#_add_libpath(){
|
|
61
|
+
# _prepend_with_colon LD_LIBRARY_PATH "${1%/}"
|
|
62
|
+
# _prepend_with_colon LD_RUN_PATH "${1%/}"
|
|
63
|
+
#}
|
|
64
|
+
#EOF
|
|
67
65
|
|
|
68
66
|
# Basic system requirements
|
|
69
67
|
# -------------------------
|
|
@@ -76,15 +74,16 @@ DEBIAN_FRONTEND=noninteractive apt-get -y install \
|
|
|
76
74
|
vim \
|
|
77
75
|
wget \
|
|
78
76
|
libc6 \
|
|
79
|
-
time numactl \
|
|
77
|
+
time numactl xvfb \
|
|
80
78
|
bison autoconf g++ libxslt1-dev make \
|
|
81
79
|
zlib1g-dev libbz2-dev libreadline-dev \
|
|
82
80
|
rsync wget curl git openssl libyaml-0-2 libyaml-dev \
|
|
83
81
|
openjdk-8-jdk \
|
|
84
82
|
libcairo2 libcairo2-dev r-base-core r-base-dev r-cran-rserve liblzma5 liblzma-dev libcurl4-openssl-dev \
|
|
85
83
|
build-essential zlib1g-dev libssl-dev libyaml-dev libffi-dev \
|
|
86
|
-
python3 python \
|
|
84
|
+
python3 python libtorch \
|
|
87
85
|
tcsh gawk \
|
|
86
|
+
ansible \
|
|
88
87
|
pandoc \
|
|
89
88
|
libtbb-dev
|
|
90
89
|
|
|
@@ -114,7 +113,8 @@ echo '_add_path "/usr/local/bin/"' >> /etc/rbbt_environment
|
|
|
114
113
|
echo 'export LC_ALL=C.UTF-8' >> /etc/rbbt_environment
|
|
115
114
|
|
|
116
115
|
|
|
117
|
-
. /etc/rbbt_environment
|
|
118
|
-
|
|
119
116
|
apt-get clean
|
|
120
117
|
rm -rf /var/lib/apt/lists/*
|
|
118
|
+
|
|
119
|
+
echo "[ -f ~/.rbbt/etc/environment ] && . ~/.rbbt/etc/environment" >> "/etc/rbbt_environment"
|
|
120
|
+
echo "source /etc/rbbt_environment" >> /etc/profile
|
|
@@ -1,19 +1,17 @@
|
|
|
1
1
|
#!/bin/bash -x
|
|
2
2
|
|
|
3
|
-
test -z $
|
|
4
|
-
test -z $
|
|
5
|
-
test -z $
|
|
3
|
+
test -z $RBBT_SERVER && export RBBT_SERVER=http://rbbt.bsc.es/
|
|
4
|
+
test -z $RBBT_FILE_SERVER && export RBBT_FILE_SERVER="$RBBT_SERVER"
|
|
5
|
+
test -z $RBBT_WORKFLOW_SERVER && export RBBT_WORKFLOW_SERVER="$RBBT_SERVER"
|
|
6
6
|
|
|
7
|
-
test -z $
|
|
8
|
-
test -z $
|
|
7
|
+
test -z $REMOTE_RESOURCES && export REMOTE_RESOURCES="Organism ICGC COSMIC KEGG InterPro Signor"
|
|
8
|
+
test -z $REMOTE_WORFLOWS && export REMOTE_WORFLOWS=""
|
|
9
9
|
|
|
10
|
-
test -z $
|
|
10
|
+
test -z $RBBT_WORKFLOW_AUTOINSTALL && export RBBT_WORKFLOW_AUTOINSTALL="true"
|
|
11
11
|
|
|
12
|
-
test -z $
|
|
12
|
+
test -z $WORKFLOWS && export WORKFLOWS=""
|
|
13
13
|
|
|
14
|
-
test -z $
|
|
15
|
-
test -z $
|
|
14
|
+
test -z $BOOTSTRAP_WORKFLOWS && export BOOTSTRAP_WORKFLOWS=""
|
|
15
|
+
test -z $BOOTSTRAP_CPUS && export BOOTSTRAP_CPUS="2"
|
|
16
16
|
|
|
17
|
-
test -z $
|
|
18
|
-
|
|
19
|
-
test -z ${RUBY_VERSION+x} && RUBY_VERSION="2.4.1"
|
|
17
|
+
test -z $RBBT_LOG && export RBBT_LOG="LOW"
|
metadata
CHANGED
|
@@ -1,14 +1,14 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: rbbt-image
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 0.1.
|
|
4
|
+
version: 0.1.40
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- Miguel Vazquez
|
|
8
8
|
autorequire:
|
|
9
9
|
bindir: bin
|
|
10
10
|
cert_chain: []
|
|
11
|
-
date:
|
|
11
|
+
date: 2023-03-28 00:00:00.000000000 Z
|
|
12
12
|
dependencies:
|
|
13
13
|
- !ruby/object:Gem::Dependency
|
|
14
14
|
name: rbbt-util
|
|
@@ -40,17 +40,25 @@ files:
|
|
|
40
40
|
- bin/run_rbbt_docker.rb
|
|
41
41
|
- lib/rbbt/docker.rb
|
|
42
42
|
- share/provision_scripts/Dockerfile.R-packages
|
|
43
|
-
- share/provision_scripts/
|
|
44
|
-
- share/provision_scripts/
|
|
43
|
+
- share/provision_scripts/R.sh
|
|
44
|
+
- share/provision_scripts/R_custom.sh
|
|
45
|
+
- share/provision_scripts/alpine_basic_setup.sh
|
|
46
|
+
- share/provision_scripts/alpine_extended_setup.sh
|
|
47
|
+
- share/provision_scripts/alpine_setup.sh
|
|
45
48
|
- share/provision_scripts/bootstrap.sh
|
|
46
|
-
- share/provision_scripts/
|
|
49
|
+
- share/provision_scripts/functions.sh
|
|
50
|
+
- share/provision_scripts/gem.sh
|
|
47
51
|
- share/provision_scripts/hacks.sh
|
|
48
|
-
- share/provision_scripts/
|
|
49
|
-
- share/provision_scripts/
|
|
50
|
-
- share/provision_scripts/
|
|
51
|
-
- share/provision_scripts/
|
|
52
|
+
- share/provision_scripts/java.sh
|
|
53
|
+
- share/provision_scripts/migrate.sh
|
|
54
|
+
- share/provision_scripts/perl_custom.sh
|
|
55
|
+
- share/provision_scripts/python.sh
|
|
56
|
+
- share/provision_scripts/python_custom.sh
|
|
57
|
+
- share/provision_scripts/ruby_custom.sh
|
|
58
|
+
- share/provision_scripts/slurm_loopback.sh
|
|
59
|
+
- share/provision_scripts/tokyocabinet.sh
|
|
52
60
|
- share/provision_scripts/ubuntu_setup.sh
|
|
53
|
-
- share/provision_scripts/
|
|
61
|
+
- share/provision_scripts/user.sh
|
|
54
62
|
- share/provision_scripts/variables.sh
|
|
55
63
|
homepage: http://github.com/mikisvaz/rbbt-image
|
|
56
64
|
licenses:
|
|
@@ -71,7 +79,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
|
71
79
|
- !ruby/object:Gem::Version
|
|
72
80
|
version: '0'
|
|
73
81
|
requirements: []
|
|
74
|
-
rubygems_version: 3.
|
|
82
|
+
rubygems_version: 3.4.8
|
|
75
83
|
signing_key:
|
|
76
84
|
specification_version: 4
|
|
77
85
|
summary: Build docker and Vagrant (VM) images
|
|
@@ -1,51 +0,0 @@
|
|
|
1
|
-
#!/bin/bash -x
|
|
2
|
-
|
|
3
|
-
# INSTALL
|
|
4
|
-
# =======
|
|
5
|
-
|
|
6
|
-
# R packages
|
|
7
|
-
# ----------
|
|
8
|
-
|
|
9
|
-
function R_install_packages(){
|
|
10
|
-
pkgs="'$1'"
|
|
11
|
-
shift
|
|
12
|
-
for p in $@; do
|
|
13
|
-
pkgs="$pkgs, '$p'"
|
|
14
|
-
done
|
|
15
|
-
echo "install.packages(c($pkgs), repos='http://cran.us.r-project.org')" | R --vanilla
|
|
16
|
-
}
|
|
17
|
-
|
|
18
|
-
function R_biocLite(){
|
|
19
|
-
pkgs="'$1'"
|
|
20
|
-
shift
|
|
21
|
-
for p in $@; do
|
|
22
|
-
echo "source('http://bioconductor.org/biocLite.R'); biocLite('$p')" | R --vanilla
|
|
23
|
-
done
|
|
24
|
-
}
|
|
25
|
-
|
|
26
|
-
function R_CMD_install(){
|
|
27
|
-
url=$1
|
|
28
|
-
name=$2
|
|
29
|
-
wget "$url" -O /tmp/R-pkg-"$name".tar.gz
|
|
30
|
-
R CMD INSTALL /tmp/R-pkg-"$name".tar.gz
|
|
31
|
-
}
|
|
32
|
-
|
|
33
|
-
R_install_packages Rcpp RJSONIO XML
|
|
34
|
-
R_CMD_install 'https://cran.r-project.org/src/contrib/Archive/plyr/plyr_1.8.1.tar.gz' plyr
|
|
35
|
-
R_CMD_install 'https://cran.r-project.org/src/contrib/Archive/car/car_2.0-22.tar.gz' car
|
|
36
|
-
R_install_packages 'digest' 'gtable' 'reshape2' 'scales' 'proto'
|
|
37
|
-
|
|
38
|
-
R_CMD_install 'https://cran.r-project.org/src/contrib/Archive/ggplot2/ggplot2_1.0.0.tar.gz' ggplot2
|
|
39
|
-
|
|
40
|
-
R_install_packages NMF Cairo drc gridSVG ggthemes mclust randomForest viper
|
|
41
|
-
|
|
42
|
-
echo 'install.packages("Rserve", repos="http://www.rforge.net/")' | R --vanilla
|
|
43
|
-
|
|
44
|
-
R_install_packages pheatmap VennDiagram Hmisc pls gridSVG
|
|
45
|
-
R_install_packages tidyverse
|
|
46
|
-
|
|
47
|
-
R_biocLite limma viper
|
|
48
|
-
|
|
49
|
-
R_install_packages pROC txtplot
|
|
50
|
-
|
|
51
|
-
rm /tmp/R-pkg-*.tar.gz
|
|
@@ -1,46 +0,0 @@
|
|
|
1
|
-
#!/bin/bash -x
|
|
2
|
-
|
|
3
|
-
# RUBY GEMS and RBBT
|
|
4
|
-
# =================
|
|
5
|
-
|
|
6
|
-
. /etc/rbbt_environment
|
|
7
|
-
|
|
8
|
-
echo "* Updating RubyGem"
|
|
9
|
-
export REALLY_GEM_UPDATE_SYSTEM=true
|
|
10
|
-
env REALLY_GEM_UPDATE_SYSTEM=true gem update --no-document --system
|
|
11
|
-
|
|
12
|
-
echo "* Installing difficult gems: ZenTest, RubyInline, rsruby and rjb"
|
|
13
|
-
gem install --force --no-document ZenTest
|
|
14
|
-
gem install --force --no-document RubyInline
|
|
15
|
-
|
|
16
|
-
# R (extra config in gem)
|
|
17
|
-
export R_INCLUDE="$(echo "$R_HOME" | sed 's@/usr/lib\(32\|64\)*@/usr/share@')/include"
|
|
18
|
-
gem install --conservative --no-document rsruby -- --with-R-dir="$R_HOME" --with-R-include="$R_INCLUDE" \
|
|
19
|
-
--with_cflags="-fPIC -g -O2 -fstack-protector --param=ssp-buffer-size=4 -Wformat -Wall -fno-strict-aliasing"
|
|
20
|
-
|
|
21
|
-
# Java (extra config in gem)
|
|
22
|
-
export JAVA_HOME=$(echo /usr/lib/jvm/java-?-openjdk-*)
|
|
23
|
-
gem install --conservative --force --no-document rjb
|
|
24
|
-
|
|
25
|
-
echo "* Installing bulk gems and Rbbt"
|
|
26
|
-
# Rbbt and some optional gems
|
|
27
|
-
gem install --no-document --force \
|
|
28
|
-
tokyocabinet \
|
|
29
|
-
ruby-prof \
|
|
30
|
-
rbbt-util rbbt-rest rbbt-dm rbbt-text rbbt-sources rbbt-phgx rbbt-GE \
|
|
31
|
-
rserve-client \
|
|
32
|
-
|
|
33
|
-
# Get good version of lockfile
|
|
34
|
-
#wget http://ubio.bioinfo.cnio.es/people/mvazquezg/lockfile-2.1.4.gem -O /tmp/lockfile-2.1.4.gem
|
|
35
|
-
echo "* Installing our version of lockfile"
|
|
36
|
-
wget http://github.com/mikisvaz/lockfile/raw/master/lockfile-2.1.5.gem -O /tmp/lockfile-2.1.5.gem
|
|
37
|
-
gem install --no-document /tmp/lockfile-2.1.5.gem
|
|
38
|
-
|
|
39
|
-
# Extra things for web interface
|
|
40
|
-
gem install --no-document --force \
|
|
41
|
-
tokyocabinet \
|
|
42
|
-
ruby-prof \
|
|
43
|
-
rbbt-util rbbt-rest rbbt-dm rbbt-text rbbt-sources rbbt-phgx rbbt-GE \
|
|
44
|
-
rserve-client \
|
|
45
|
-
uglifier therubyracer kramdown pandoc pandoc-ruby\
|
|
46
|
-
puma prawn prawn-svg
|
|
File without changes
|
|
File without changes
|