openshift-origin-node 1.3.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of openshift-origin-node might be problematic. Click here for more details.
- data/COPYRIGHT +1 -0
- data/Gemfile +4 -0
- data/LICENSE +11 -0
- data/README.md +3 -0
- data/Rakefile +28 -0
- data/bin/oo-add-alias +93 -0
- data/bin/oo-app-create +110 -0
- data/bin/oo-app-destroy +100 -0
- data/bin/oo-app-state-show +74 -0
- data/bin/oo-authorized-ssh-key-add +83 -0
- data/bin/oo-authorized-ssh-key-remove +82 -0
- data/bin/oo-broker-auth-key-add +84 -0
- data/bin/oo-broker-auth-key-remove +72 -0
- data/bin/oo-cartridge-info +70 -0
- data/bin/oo-cartridge-list +70 -0
- data/bin/oo-connector-execute +94 -0
- data/bin/oo-env-var-add +81 -0
- data/bin/oo-env-var-remove +78 -0
- data/bin/oo-get-quota +64 -0
- data/bin/oo-remove-alias +93 -0
- data/bin/oo-set-quota +59 -0
- data/conf/node.conf +30 -0
- data/conf/resource_limits.template +67 -0
- data/lib/openshift-origin-node.rb +29 -0
- data/lib/openshift-origin-node/config.rb +21 -0
- data/lib/openshift-origin-node/environment.rb +26 -0
- data/lib/openshift-origin-node/model/application_container.rb +298 -0
- data/lib/openshift-origin-node/model/frontend_httpd.rb +346 -0
- data/lib/openshift-origin-node/model/node.rb +134 -0
- data/lib/openshift-origin-node/model/unix_user.rb +738 -0
- data/lib/openshift-origin-node/plugins/unix_user_observer.rb +86 -0
- data/lib/openshift-origin-node/utils/shell_exec.rb +115 -0
- data/lib/openshift-origin-node/version.rb +23 -0
- data/misc/bin/oo-admin-ctl-cgroups +482 -0
- data/misc/bin/oo-cgroup-read +25 -0
- data/misc/bin/oo-get-mcs-level +29 -0
- data/misc/bin/oo-trap-user +248 -0
- data/misc/bin/rhcsh +155 -0
- data/misc/bin/setup_pam_fs_limits.sh +146 -0
- data/misc/bin/teardown_pam_fs_limits.sh +73 -0
- data/misc/doc/cgconfig.conf +26 -0
- data/misc/etc/openshift-run.conf +1 -0
- data/misc/init/openshift-cgroups +56 -0
- data/misc/services/openshift-cgroups.service +14 -0
- data/openshift-origin-node.gemspec +31 -0
- data/rubygem-openshift-origin-node.spec +263 -0
- data/test/test_helper.rb +20 -0
- data/test/unit/frontend_httpd_test.rb +144 -0
- data/test/unit/unix_user_test.rb +95 -0
- data/test/unit/version_test.rb +45 -0
- metadata +230 -0
@@ -0,0 +1,134 @@
|
|
1
|
+
#--
|
2
|
+
# Copyright 2012 Red Hat, Inc.
|
3
|
+
#
|
4
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
5
|
+
# you may not use this file except in compliance with the License.
|
6
|
+
# You may obtain a copy of the License at
|
7
|
+
#
|
8
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
9
|
+
#
|
10
|
+
# Unless required by applicable law or agreed to in writing, software
|
11
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
12
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
13
|
+
# See the License for the specific language governing permissions and
|
14
|
+
# limitations under the License.
|
15
|
+
#++
|
16
|
+
|
17
|
+
require 'openshift-origin-node'
|
18
|
+
require 'openshift-origin-common'
|
19
|
+
require 'systemu'
|
20
|
+
|
21
|
+
module OpenShift
|
22
|
+
class NodeCommandException < StandardError; end
|
23
|
+
|
24
|
+
class Node < Model
|
25
|
+
def self.get_cartridge_list(list_descriptors = false, porcelain = false, oo_debug = false)
|
26
|
+
carts = []
|
27
|
+
|
28
|
+
cartridge_path = OpenShift::Config.new.get("CARTRIDGE_BASE_PATH")
|
29
|
+
Dir.foreach(cartridge_path) do |cart_dir|
|
30
|
+
next if [".", "..", "embedded", "abstract", "abstract-httpd", "abstract-jboss"].include? cart_dir
|
31
|
+
path = File.join(cartridge_path, cart_dir, "info", "manifest.yml")
|
32
|
+
begin
|
33
|
+
print "Loading #{cart_dir}..." if oo_debug
|
34
|
+
carts.push OpenShift::Cartridge.new.from_descriptor(YAML.load(File.open(path)))
|
35
|
+
print "OK\n" if oo_debug
|
36
|
+
rescue Exception => e
|
37
|
+
print "ERROR\n" if oo_debug
|
38
|
+
print "#{e.message}\n#{e.backtrace.inspect}\n" unless porcelain
|
39
|
+
end
|
40
|
+
end
|
41
|
+
|
42
|
+
print "\n\n\n" if oo_debug
|
43
|
+
|
44
|
+
output = ""
|
45
|
+
if porcelain
|
46
|
+
if list_descriptors
|
47
|
+
output << "CLIENT_RESULT: "
|
48
|
+
output << carts.map{|c| c.to_descriptor.to_yaml}.to_json
|
49
|
+
else
|
50
|
+
output << "CLIENT_RESULT: "
|
51
|
+
output << carts.map{|c| c.name}.to_json
|
52
|
+
end
|
53
|
+
else
|
54
|
+
if list_descriptors
|
55
|
+
carts.each do |c|
|
56
|
+
output << "Cartridge name: #{c.name}\n\nDescriptor:\n #{c.to_descriptor.inspect}\n\n\n"
|
57
|
+
end
|
58
|
+
else
|
59
|
+
output << "Cartridges:\n"
|
60
|
+
carts.each do |c|
|
61
|
+
output << "\t#{c.name}\n"
|
62
|
+
end
|
63
|
+
end
|
64
|
+
end
|
65
|
+
output
|
66
|
+
end
|
67
|
+
|
68
|
+
def self.get_cartridge_info(cart_name, porcelain = false, oo_debug = false)
|
69
|
+
output = ""
|
70
|
+
cart_found = false
|
71
|
+
|
72
|
+
cartridge_path = OpenShift::Config.new.get("CARTRIDGE_BASE_PATH")
|
73
|
+
Dir.foreach(cartridge_path) do |cart_dir|
|
74
|
+
next if [".", "..", "embedded", "abstract", "abstract-httpd", "haproxy-1.4", "mysql-5.1", "mongodb-2.2", "postgresql-8.4"].include? cart_dir
|
75
|
+
path = File.join(cartridge_path, cart_dir, "info", "manifest.yml")
|
76
|
+
begin
|
77
|
+
cart = OpenShift::Cartridge.new.from_descriptor(YAML.load(File.open(path)))
|
78
|
+
if cart.name == cart_name
|
79
|
+
output << "CLIENT_RESULT: "
|
80
|
+
output << cart.to_descriptor.to_json
|
81
|
+
cart_found = true
|
82
|
+
break
|
83
|
+
end
|
84
|
+
rescue Exception => e
|
85
|
+
print "ERROR\n" if oo_debug
|
86
|
+
print "#{e.message}\n#{e.backtrace.inspect}\n" unless porcelain
|
87
|
+
end
|
88
|
+
end
|
89
|
+
|
90
|
+
embedded_cartridge_path = File.join(cartridge_path, "embedded")
|
91
|
+
if (! cart_found) and File.directory?(embedded_cartridge_path)
|
92
|
+
Dir.foreach(embedded_cartridge_path) do |cart_dir|
|
93
|
+
next if [".",".."].include? cart_dir
|
94
|
+
path = File.join(embedded_cartridge_path, cart_dir, "info", "manifest.yml")
|
95
|
+
begin
|
96
|
+
cart = OpenShift::Cartridge.new.from_descriptor(YAML.load(File.open(path)))
|
97
|
+
if cart.name == cart_name
|
98
|
+
output << "CLIENT_RESULT: "
|
99
|
+
output << cart.to_descriptor.to_json
|
100
|
+
break
|
101
|
+
end
|
102
|
+
rescue Exception => e
|
103
|
+
print "ERROR\n" if oo_debug
|
104
|
+
print "#{e.message}\n#{e.backtrace.inspect}\n" unless porcelain
|
105
|
+
end
|
106
|
+
end
|
107
|
+
end
|
108
|
+
output
|
109
|
+
end
|
110
|
+
|
111
|
+
def self.get_quota(uuid)
|
112
|
+
cmd = %"a -w #{uuid} | awk '/^.*\\/dev/ {print $1":"$2":"$3":"$4":"$5":"$6":"$7}'; exit ${PIPESTATUS[0]}&
|
113
|
+
st, out, errout = systemu cmd
|
114
|
+
if st.exitstatus == 0 || st.exitstatus == 1
|
115
|
+
arr = out.strip.split(":")
|
116
|
+
raise NodeCommandException.new "Error: #{errout} executing command #{cmd}" unless arr.length == 7
|
117
|
+
arr
|
118
|
+
else
|
119
|
+
raise NodeCommandException.new "Error: #{errout} executing command #{cmd}"
|
120
|
+
end
|
121
|
+
end
|
122
|
+
|
123
|
+
def self.set_quota(uuid, blocksmax, inodemax)
|
124
|
+
cur_quota = get_quota(uuid)
|
125
|
+
inodemax = cur_quota[6] if inodemax.to_s.empty?
|
126
|
+
|
127
|
+
mountpoint = %x[quota -w #{uuid} | awk '/^.*\\/dev/ {print $1}']
|
128
|
+
cmd = "setquota -u #{uuid} 0 #{blocksmax} 0 #{inodemax} -a #{mountpoint}"
|
129
|
+
st, out, errout = systemu cmd
|
130
|
+
raise NodeCommandException.new "Error: #{errout} executing command #{cmd}" unless st.exitstatus == 0
|
131
|
+
end
|
132
|
+
|
133
|
+
end
|
134
|
+
end
|
@@ -0,0 +1,738 @@
|
|
1
|
+
#--
|
2
|
+
# Copyright 2010 Red Hat, Inc.
|
3
|
+
#
|
4
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
5
|
+
# you may not use this file except in compliance with the License.
|
6
|
+
# You may obtain a copy of the License at
|
7
|
+
#
|
8
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
9
|
+
#
|
10
|
+
# Unless required by applicable law or agreed to in writing, software
|
11
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
12
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
13
|
+
# See the License for the specific language governing permissions and
|
14
|
+
# limitations under the License.
|
15
|
+
#++
|
16
|
+
|
17
|
+
require 'rubygems'
|
18
|
+
require 'openshift-origin-node/utils/shell_exec'
|
19
|
+
require 'openshift-origin-node/model/frontend_httpd.rb'
|
20
|
+
require 'openshift-origin-common'
|
21
|
+
require 'syslog'
|
22
|
+
require 'fcntl'
|
23
|
+
|
24
|
+
module OpenShift
|
25
|
+
class UserCreationException < Exception
|
26
|
+
end
|
27
|
+
|
28
|
+
class UserDeletionException < Exception
|
29
|
+
end
|
30
|
+
|
31
|
+
# == Unix User
|
32
|
+
#
|
33
|
+
# Represents a user account on the system.
|
34
|
+
class UnixUser < Model
|
35
|
+
include OpenShift::Utils::ShellExec
|
36
|
+
attr_reader :uuid, :uid, :gid, :gecos, :homedir, :application_uuid,
|
37
|
+
:container_uuid, :app_name, :namespace, :quota_blocks, :quota_files,
|
38
|
+
:container_name
|
39
|
+
attr_accessor :debug
|
40
|
+
|
41
|
+
DEFAULT_SKEL_DIR = File.join(OpenShift::Config::CONF_DIR,"skel")
|
42
|
+
|
43
|
+
def initialize(application_uuid, container_uuid, user_uid=nil,
|
44
|
+
app_name=nil, container_name=nil, namespace=nil, quota_blocks=nil, quota_files=nil, debug=false)
|
45
|
+
Syslog.open('openshift-origin-node', Syslog::LOG_PID, Syslog::LOG_LOCAL0) unless Syslog.opened?
|
46
|
+
|
47
|
+
@config = OpenShift::Config.new
|
48
|
+
|
49
|
+
@container_uuid = container_uuid
|
50
|
+
@application_uuid = application_uuid
|
51
|
+
@uuid = container_uuid
|
52
|
+
@app_name = app_name
|
53
|
+
@container_name = container_name
|
54
|
+
@namespace = namespace
|
55
|
+
@quota_blocks = quota_blocks
|
56
|
+
@quota_files = quota_files
|
57
|
+
@debug = debug
|
58
|
+
|
59
|
+
begin
|
60
|
+
user_info = Etc.getpwnam(@uuid)
|
61
|
+
@uid = user_info.uid
|
62
|
+
@gid = user_info.gid
|
63
|
+
@gecos = user_info.gecos
|
64
|
+
@homedir = "#{user_info.dir}/"
|
65
|
+
rescue ArgumentError => e
|
66
|
+
@uid = user_uid
|
67
|
+
@gid = user_uid
|
68
|
+
@gecos = nil
|
69
|
+
@homedir = nil
|
70
|
+
end
|
71
|
+
end
|
72
|
+
|
73
|
+
def name
|
74
|
+
@uuid
|
75
|
+
end
|
76
|
+
|
77
|
+
# Public: Create an empty gear.
|
78
|
+
#
|
79
|
+
# Examples
|
80
|
+
#
|
81
|
+
# create
|
82
|
+
# # => nil
|
83
|
+
# # a user
|
84
|
+
# # Setup permissions
|
85
|
+
#
|
86
|
+
# Returns nil on Success or raises on Failure
|
87
|
+
def create
|
88
|
+
skel_dir = @config.get("GEAR_SKEL_DIR") || DEFAULT_SKEL_DIR
|
89
|
+
shell = @config.get("GEAR_SHELL") || "/bin/bash"
|
90
|
+
gecos = @config.get("GEAR_GECOS") || "OO application container"
|
91
|
+
notify_observers(:before_unix_user_create)
|
92
|
+
basedir = @config.get("GEAR_BASE_DIR")
|
93
|
+
|
94
|
+
# lock to prevent race condition between create and delete of gear
|
95
|
+
uuid_lock_file = "/var/lock/oo-create.#{@uuid}"
|
96
|
+
File.open(uuid_lock_file, File::RDWR|File::CREAT, 0o0600) do | uuid_lock |
|
97
|
+
uuid_lock.fcntl(Fcntl::F_SETFD, Fcntl::FD_CLOEXEC)
|
98
|
+
uuid_lock.flock(File::LOCK_EX)
|
99
|
+
|
100
|
+
# Lock to prevent race condition on obtaining a UNIX user uid.
|
101
|
+
# When running without districts, there is a simple search on the
|
102
|
+
# passwd file for the next available uid.
|
103
|
+
File.open("/var/lock/oo-create", File::RDWR|File::CREAT, 0o0600) do | uid_lock |
|
104
|
+
uid_lock.fcntl(Fcntl::F_SETFD, Fcntl::FD_CLOEXEC)
|
105
|
+
uid_lock.flock(File::LOCK_EX)
|
106
|
+
|
107
|
+
unless @uid
|
108
|
+
@uid = @gid = next_uid
|
109
|
+
end
|
110
|
+
|
111
|
+
unless @homedir
|
112
|
+
@homedir = File.join(basedir,@uuid)
|
113
|
+
end
|
114
|
+
|
115
|
+
cmd = %{useradd -u #{@uid} \
|
116
|
+
-d #{@homedir} \
|
117
|
+
-s #{shell} \
|
118
|
+
-c '#{gecos}' \
|
119
|
+
-m \
|
120
|
+
-k #{skel_dir} \
|
121
|
+
#{@uuid}}
|
122
|
+
out,err,rc = shellCmd(cmd)
|
123
|
+
raise UserCreationException.new(
|
124
|
+
"ERROR: unable to create user account(#{rc}): #{cmd.squeeze(" ")} stdout: #{out} stderr: #{err}"
|
125
|
+
) unless rc == 0
|
126
|
+
|
127
|
+
FileUtils.chown("root", @uuid, @homedir)
|
128
|
+
FileUtils.chmod 0o0750, @homedir
|
129
|
+
|
130
|
+
if @config.get("CREATE_APP_SYMLINKS").to_i == 1
|
131
|
+
unobfuscated = File.join(File.dirname(@homedir),"#{@container_name}-#{namespace}")
|
132
|
+
if not File.exists? unobfuscated
|
133
|
+
FileUtils.ln_s File.basename(@homedir), unobfuscated, :force=>true
|
134
|
+
end
|
135
|
+
end
|
136
|
+
end
|
137
|
+
notify_observers(:after_unix_user_create)
|
138
|
+
initialize_homedir(basedir, @homedir, @config.get("CARTRIDGE_BASE_PATH"))
|
139
|
+
initialize_openshift_port_proxy
|
140
|
+
|
141
|
+
uuid_lock.flock(File::LOCK_UN)
|
142
|
+
File.unlink(uuid_lock_file)
|
143
|
+
end
|
144
|
+
end
|
145
|
+
|
146
|
+
# Public: Destroys a gear stopping all processes and removing all files
|
147
|
+
#
|
148
|
+
# The order of the calls and gyrations done in this code is to prevent
|
149
|
+
# pam_namespace from locking polyinstantiated directories during
|
150
|
+
# their deletion. If you see "broken" gears, i.e. ~uuid/.tmp and
|
151
|
+
# ~/uuid/.sandbox after #destroy has been called, this method is broken.
|
152
|
+
# See Bug 853582 for history.
|
153
|
+
#
|
154
|
+
# Examples
|
155
|
+
#
|
156
|
+
# destroy
|
157
|
+
# # => nil
|
158
|
+
#
|
159
|
+
# Returns nil on Success or raises on Failure
|
160
|
+
def destroy
|
161
|
+
if @uid.nil? and (not File.directory?(@homedir.to_s))
|
162
|
+
# gear seems to have been destroyed already... suppress any error
|
163
|
+
# TODO : remove remaining stuff if it exists, e.g. .httpd/#{uuid}* etc
|
164
|
+
return nil
|
165
|
+
end
|
166
|
+
raise UserDeletionException.new(
|
167
|
+
"ERROR: unable to destroy user account #{@uuid}"
|
168
|
+
) if @uid.nil? || @homedir.nil? || @uuid.nil?
|
169
|
+
|
170
|
+
# Don't try to delete a gear that is being scaled-up|created|deleted
|
171
|
+
uuid_lock_file = "/var/lock/oo-create.#{@uuid}"
|
172
|
+
File.open(uuid_lock_file, File::RDWR|File::CREAT, 0o0600) do | lock |
|
173
|
+
lock.fcntl(Fcntl::F_SETFD, Fcntl::FD_CLOEXEC)
|
174
|
+
lock.flock(File::LOCK_EX)
|
175
|
+
|
176
|
+
# These calls and their order is designed to release pam_namespace's
|
177
|
+
# locks on .tmp and .sandbox. Change then at your peril.
|
178
|
+
#
|
179
|
+
# 1. Kill off the easy processes
|
180
|
+
# 2. Lock down the user from creating new processes (cgroups freeze, nprocs 0)
|
181
|
+
# 3. Attempt to move any processes that didn't die into state 'D' (re: cgroups freeze)
|
182
|
+
self.class.kill_procs(@uid)
|
183
|
+
notify_observers(:before_unix_user_destroy)
|
184
|
+
self.class.kill_procs(@uid)
|
185
|
+
|
186
|
+
purge_sysvipc(uuid)
|
187
|
+
initialize_openshift_port_proxy
|
188
|
+
|
189
|
+
if @config.get("CREATE_APP_SYMLINKS").to_i == 1
|
190
|
+
Dir.foreach(File.dirname(@homedir)) do |dent|
|
191
|
+
unobfuscate = File.join(File.dirname(@homedir), dent)
|
192
|
+
if (File.symlink?(unobfuscate)) &&
|
193
|
+
(File.readlink(unobfuscate) == File.basename(@homedir))
|
194
|
+
File.unlink(unobfuscate)
|
195
|
+
end
|
196
|
+
end
|
197
|
+
end
|
198
|
+
|
199
|
+
OpenShift::FrontendHttpServer.new(@container_uuid,@container_name,@namespace).destroy
|
200
|
+
|
201
|
+
dirs = list_home_dir(@homedir)
|
202
|
+
cmd = "userdel -f \"#{@uuid}\""
|
203
|
+
out,err,rc = shellCmd(cmd)
|
204
|
+
raise UserDeletionException.new(
|
205
|
+
"ERROR: unable to destroy user account(#{rc}): #{cmd} stdout: #{out} stderr: #{err}"
|
206
|
+
) unless rc == 0
|
207
|
+
|
208
|
+
# 1. Don't believe everything you read on the userdel man page...
|
209
|
+
# 2. If there are any active processes left pam_namespace is not going
|
210
|
+
# to let polyinstantiated directories be deleted.
|
211
|
+
FileUtils.rm_rf(@homedir)
|
212
|
+
if File.exists?(@homedir)
|
213
|
+
# Ops likes the verbose verbage
|
214
|
+
Syslog.alert %Q{
|
215
|
+
1st attempt to remove \'#{@homedir}\' from filesystem failed.
|
216
|
+
Dir(before) #{@uuid}/#{@uid} => #{dirs}
|
217
|
+
Dir(after) #{@uuid}/#{@uid} => #{list_home_dir(@homedir)}
|
218
|
+
}
|
219
|
+
end
|
220
|
+
|
221
|
+
# release resources (cgroups thaw), this causes Zombies to get killed
|
222
|
+
notify_observers(:after_unix_user_destroy)
|
223
|
+
|
224
|
+
# try one last time...
|
225
|
+
if File.exists?(@homedir)
|
226
|
+
sleep(5) # don't fear the reaper
|
227
|
+
FileUtils.rm_rf(@homedir) # This is our last chance to nuke the polyinstantiated directories
|
228
|
+
Syslog.alert "2nd attempt to remove \'#{@homedir}\' from filesystem failed." if File.exists?(@homedir)
|
229
|
+
end
|
230
|
+
|
231
|
+
lock.flock(File::LOCK_UN)
|
232
|
+
File.unlink(uuid_lock_file)
|
233
|
+
end
|
234
|
+
end
|
235
|
+
|
236
|
+
# Public: Append an SSH key to a users authorized_keys file
|
237
|
+
#
|
238
|
+
# key - The String value of the ssh key.
|
239
|
+
# key_type - The String value of the key type ssh-(rsa|dss)).
|
240
|
+
# comment - The String value of the comment to append to the key.
|
241
|
+
#
|
242
|
+
# Examples
|
243
|
+
#
|
244
|
+
# add_ssh_key('AAAAB3NzaC1yc2EAAAADAQABAAABAQDE0DfenPIHn5Bq/...',
|
245
|
+
# 'ssh-rsa',
|
246
|
+
# 'example@example.com')
|
247
|
+
# # => nil
|
248
|
+
#
|
249
|
+
# Returns nil on Success or raises on Failure
|
250
|
+
def add_ssh_key(key, key_type=nil, comment=nil)
|
251
|
+
comment = "" unless comment
|
252
|
+
self.class.notify_observers(:before_add_ssh_key, self, key)
|
253
|
+
|
254
|
+
authorized_keys_file = File.join(@homedir, ".ssh", "authorized_keys")
|
255
|
+
keys = read_ssh_keys authorized_keys_file
|
256
|
+
key_type = "ssh-rsa" if key_type.to_s.strip.length == 0
|
257
|
+
cloud_name = @config.get("CLOUD_NAME") || "OPENSHIFT"
|
258
|
+
ssh_comment = "#{cloud_name}-#{@uuid}#{comment}"
|
259
|
+
shell = @config.get("GEAR_SHELL") || "/bin/bash"
|
260
|
+
cmd_entry = "command=\"#{shell}\",no-X11-forwarding #{key_type} #{key} #{ssh_comment}"
|
261
|
+
|
262
|
+
keys[ssh_comment] = cmd_entry
|
263
|
+
write_ssh_keys authorized_keys_file, keys
|
264
|
+
|
265
|
+
self.class.notify_observers(:after_add_ssh_key, self, key)
|
266
|
+
end
|
267
|
+
|
268
|
+
# Public: Remove an SSH key from a users authorized_keys file.
|
269
|
+
#
|
270
|
+
# key - The String value of the ssh key.
|
271
|
+
# comment - The String value of the comment associated with the key.
|
272
|
+
#
|
273
|
+
# Examples
|
274
|
+
#
|
275
|
+
# remove_ssh_key('AAAAB3NzaC1yc2EAAAADAQABAAABAQDE0DfenPIHn5Bq/...',
|
276
|
+
# 'example@example.com')
|
277
|
+
# # => nil
|
278
|
+
#
|
279
|
+
# Returns nil on Success or raises on Failure
|
280
|
+
def remove_ssh_key(key, comment=nil)
|
281
|
+
self.class.notify_observers(:before_remove_ssh_key, self, key)
|
282
|
+
|
283
|
+
authorized_keys_file = File.join(@homedir, ".ssh", "authorized_keys")
|
284
|
+
keys = read_ssh_keys authorized_keys_file
|
285
|
+
|
286
|
+
if comment
|
287
|
+
keys.delete_if{ |k, v| v.include?(key) && v.include?(comment)}
|
288
|
+
else
|
289
|
+
keys.delete_if{ |k, v| v.include?(key)}
|
290
|
+
end
|
291
|
+
|
292
|
+
write_ssh_keys authorized_keys_file, keys
|
293
|
+
|
294
|
+
self.class.notify_observers(:after_remove_ssh_key, self, key)
|
295
|
+
end
|
296
|
+
|
297
|
+
# Public: Add an environment variable to a given gear.
|
298
|
+
#
|
299
|
+
# key - The String value of target environment variable.
|
300
|
+
# value - The String value to place inside the environment variable.
|
301
|
+
# prefix_cloud_name - The String value to append in front of key.
|
302
|
+
#
|
303
|
+
# Examples
|
304
|
+
#
|
305
|
+
# add_env_var('mysql-5.3')
|
306
|
+
# # => 36
|
307
|
+
#
|
308
|
+
# Returns the Integer value for how many bytes got written or raises on
|
309
|
+
# failure.
|
310
|
+
def add_env_var(key, value, prefix_cloud_name = false, &blk)
|
311
|
+
env_dir = File.join(@homedir,'.env/')
|
312
|
+
if prefix_cloud_name
|
313
|
+
key = (@config.get('CLOUD_NAME') || 'OPENSHIFT') + "_#{key}"
|
314
|
+
end
|
315
|
+
filename = File.join(env_dir, key)
|
316
|
+
File.open(filename,
|
317
|
+
File::WRONLY|File::TRUNC|File::CREAT) do |file|
|
318
|
+
file.write "export #{key}='#{value}'"
|
319
|
+
end
|
320
|
+
|
321
|
+
if block_given?
|
322
|
+
blk.call(value)
|
323
|
+
end
|
324
|
+
end
|
325
|
+
|
326
|
+
# Public: list directories (cartridges) in home directory
|
327
|
+
# @param [String] home directory
|
328
|
+
# @return [String] comma separated list of directories
|
329
|
+
def list_home_dir(home_dir)
|
330
|
+
results = []
|
331
|
+
Dir.foreach(home_dir) do |entry|
|
332
|
+
#next if entry =~ /^\.{1,2}/ # Ignore ".", "..", or hidden files
|
333
|
+
results << entry
|
334
|
+
end
|
335
|
+
results.join(', ')
|
336
|
+
end
|
337
|
+
|
338
|
+
# Public: Remove an environment variable from a given gear.
|
339
|
+
#
|
340
|
+
# key - String name of the environment variable to remove.
|
341
|
+
# prefix_cloud_name - String prefix to append to key.
|
342
|
+
#
|
343
|
+
# Examples
|
344
|
+
#
|
345
|
+
# remove_env_var('OPENSHIFT_MONGODB_DB_URL')
|
346
|
+
# # => nil
|
347
|
+
#
|
348
|
+
# Returns an nil on success and false on failure.
|
349
|
+
def remove_env_var(key, prefix_cloud_name=false)
|
350
|
+
status = false
|
351
|
+
[".env", ".env/.uservars"].each do |path|
|
352
|
+
env_dir = File.join(@homedir,path)
|
353
|
+
if prefix_cloud_name
|
354
|
+
key = (@config.get("CLOUD_NAME") || "OPENSHIFT") + "_#{key}"
|
355
|
+
end
|
356
|
+
env_file_path = File.join(env_dir, key)
|
357
|
+
FileUtils.rm_f env_file_path
|
358
|
+
status = status ? true : (File.exists?(env_file_path) ? false : true)
|
359
|
+
end
|
360
|
+
status
|
361
|
+
end
|
362
|
+
|
363
|
+
# Public: Add broker authorization keys so gear can communicate with
|
364
|
+
# broker.
|
365
|
+
#
|
366
|
+
# iv - A String value for the IV file.
|
367
|
+
# token - A String value for the token file.
|
368
|
+
#
|
369
|
+
# Examples
|
370
|
+
# add_broker_auth('ivvalue', 'tokenvalue')
|
371
|
+
# # => ["/var/lib/openshift/UUID/.auth/iv",
|
372
|
+
# "/var/lib/openshift/UUID/.auth/token"]
|
373
|
+
#
|
374
|
+
# Returns An Array of Strings for the newly created auth files
|
375
|
+
def add_broker_auth(iv,token)
|
376
|
+
broker_auth_dir=File.join(@homedir,'.auth')
|
377
|
+
FileUtils.mkdir_p broker_auth_dir
|
378
|
+
File.open(File.join(broker_auth_dir, 'iv'),
|
379
|
+
File::WRONLY|File::TRUNC|File::CREAT) do |file|
|
380
|
+
file.write iv
|
381
|
+
end
|
382
|
+
File.open(File.join(broker_auth_dir, 'token'),
|
383
|
+
File::WRONLY|File::TRUNC|File::CREAT) do |file|
|
384
|
+
file.write token
|
385
|
+
end
|
386
|
+
|
387
|
+
FileUtils.chown_R("root", @uuid,broker_auth_dir)
|
388
|
+
FileUtils.chmod(0o0750, broker_auth_dir)
|
389
|
+
FileUtils.chmod(0o0640, Dir.glob("#{broker_auth_dir}/*"))
|
390
|
+
end
|
391
|
+
|
392
|
+
# Public: Remove broker authentication keys from gear.
|
393
|
+
#
|
394
|
+
# Examples
|
395
|
+
# remove_broker_auth
|
396
|
+
# # => nil
|
397
|
+
#
|
398
|
+
# Returns nil on Success and false on Failure
|
399
|
+
def remove_broker_auth
|
400
|
+
broker_auth_dir=File.join(@homedir, '.auth')
|
401
|
+
FileUtils.rm_rf broker_auth_dir
|
402
|
+
File.exists?(broker_auth_dir) ? false : true
|
403
|
+
end
|
404
|
+
|
405
|
+
#private
|
406
|
+
|
407
|
+
# Private: Create and populate the users home dir.
|
408
|
+
#
|
409
|
+
# Examples
|
410
|
+
# initialize_homedir
|
411
|
+
# # => nil
|
412
|
+
# # Creates:
|
413
|
+
# # ~
|
414
|
+
# # ~/.tmp/
|
415
|
+
# # ~/.sandbox/$uuid
|
416
|
+
# # ~/.env/
|
417
|
+
# # APP_UUID, GEAR_UUID, APP_NAME, APP_DNS, HOMEDIR, DATA_DIR, \
|
418
|
+
# # GEAR_DNS, GEAR_NAME, PATH, REPO_DIR, TMP_DIR, HISTFILE
|
419
|
+
# # ~/app-root
|
420
|
+
# # ~/app-root/data
|
421
|
+
# # ~/app-root/runtime/repo
|
422
|
+
# # ~/app-root/repo -> runtime/repo
|
423
|
+
# # ~/app-root/runtime/data -> ../data
|
424
|
+
#
|
425
|
+
# Returns nil on Success and raises on Failure.
|
426
|
+
def initialize_homedir(basedir, homedir, cart_basedir)
|
427
|
+
@homedir = homedir
|
428
|
+
notify_observers(:before_initialize_homedir)
|
429
|
+
homedir = homedir.end_with?('/') ? homedir : homedir + '/'
|
430
|
+
|
431
|
+
tmp_dir = File.join(homedir, ".tmp")
|
432
|
+
# Required for polyinstantiated tmp dirs to work
|
433
|
+
FileUtils.mkdir_p tmp_dir
|
434
|
+
FileUtils.chmod(0o0000, tmp_dir)
|
435
|
+
|
436
|
+
sandbox_dir = File.join(homedir, ".sandbox")
|
437
|
+
FileUtils.mkdir_p sandbox_dir
|
438
|
+
FileUtils.chmod(0o0000, sandbox_dir)
|
439
|
+
|
440
|
+
sandbox_uuid_dir = File.join(sandbox_dir, @uuid)
|
441
|
+
FileUtils.mkdir_p sandbox_uuid_dir
|
442
|
+
FileUtils.chmod(0o1755, sandbox_uuid_dir)
|
443
|
+
|
444
|
+
env_dir = File.join(homedir, ".env")
|
445
|
+
FileUtils.mkdir_p(env_dir)
|
446
|
+
FileUtils.chmod(0o0750, env_dir)
|
447
|
+
FileUtils.chown(nil, @uuid, env_dir)
|
448
|
+
|
449
|
+
ssh_dir = File.join(homedir, ".ssh")
|
450
|
+
FileUtils.mkdir_p(ssh_dir)
|
451
|
+
FileUtils.chmod(0o0750, ssh_dir)
|
452
|
+
FileUtils.chown(nil, @uuid, ssh_dir)
|
453
|
+
|
454
|
+
geardir = File.join(homedir, @container_name, "/")
|
455
|
+
gearappdir = File.join(homedir, "app-root", "/")
|
456
|
+
|
457
|
+
add_env_var("APP_DNS",
|
458
|
+
"#{@app_name}-#{@namespace}.#{@config.get("CLOUD_DOMAIN")}",
|
459
|
+
true)
|
460
|
+
add_env_var("APP_NAME", @app_name, true)
|
461
|
+
add_env_var("APP_UUID", @application_uuid, true)
|
462
|
+
|
463
|
+
data_dir = File.join(gearappdir, "data", "/")
|
464
|
+
add_env_var("DATA_DIR", data_dir, true) {|v|
|
465
|
+
FileUtils.mkdir_p(v, :verbose => @debug)
|
466
|
+
}
|
467
|
+
add_env_var("HISTFILE", File.join(data_dir, ".bash_history"))
|
468
|
+
profile = File.join(data_dir, ".bash_profile")
|
469
|
+
File.open(profile, File::WRONLY|File::TRUNC|File::CREAT, 0o0600) {|file|
|
470
|
+
file.write %Q{
|
471
|
+
# Warning: Be careful with modifications to this file,
|
472
|
+
# Your changes may cause your application to fail.
|
473
|
+
}
|
474
|
+
}
|
475
|
+
FileUtils.chown(@uuid, @uuid, profile, :verbose => @debug)
|
476
|
+
|
477
|
+
|
478
|
+
add_env_var("GEAR_DNS",
|
479
|
+
"#{@container_name}-#{@namespace}.#{@config.get("CLOUD_DOMAIN")}",
|
480
|
+
true)
|
481
|
+
add_env_var("GEAR_NAME", @container_name, true)
|
482
|
+
add_env_var("GEAR_UUID", @container_uuid, true)
|
483
|
+
|
484
|
+
add_env_var("HOMEDIR", homedir, true)
|
485
|
+
|
486
|
+
add_env_var("PATH",
|
487
|
+
"#{cart_basedir}abstract-httpd/info/bin/:#{cart_basedir}abstract/info/bin/:$PATH",
|
488
|
+
false)
|
489
|
+
|
490
|
+
add_env_var("REPO_DIR", File.join(gearappdir, "runtime", "repo", "/"), true) {|v|
|
491
|
+
FileUtils.mkdir_p(v, :verbose => @debug)
|
492
|
+
FileUtils.cd gearappdir do |d|
|
493
|
+
FileUtils.ln_s("runtime/repo", "repo", :verbose => @debug)
|
494
|
+
end
|
495
|
+
FileUtils.cd File.join(gearappdir, "runtime") do |d|
|
496
|
+
FileUtils.ln_s("../data", "data", :verbose => @debug)
|
497
|
+
end
|
498
|
+
}
|
499
|
+
|
500
|
+
add_env_var("TMP_DIR", "/tmp/", true)
|
501
|
+
|
502
|
+
# Update all directory entries ~/app-root/*
|
503
|
+
Dir[gearappdir + "/*"].entries.reject{|e| [".", ".."].include? e}.each {|e|
|
504
|
+
FileUtils.chmod_R(0o0750, e, :verbose => @debug)
|
505
|
+
FileUtils.chown_R(@uuid, @uuid, e, :verbose => @debug)
|
506
|
+
}
|
507
|
+
FileUtils.chown(nil, @uuid, gearappdir, :verbose => @debug)
|
508
|
+
raise "Failed to instantiate gear: missing application directory (#{gearappdir})" unless File.exist?(gearappdir)
|
509
|
+
|
510
|
+
state_file = File.join(gearappdir, "runtime", ".state")
|
511
|
+
File.open(state_file, File::WRONLY|File::TRUNC|File::CREAT, 0o0660) {|file|
|
512
|
+
file.write "new\n"
|
513
|
+
}
|
514
|
+
FileUtils.chown(@uuid, @uuid, state_file, :verbose => @debug)
|
515
|
+
|
516
|
+
OpenShift::FrontendHttpServer.new(@container_uuid,@container_name,@namespace).create
|
517
|
+
|
518
|
+
# Fix SELinux context
|
519
|
+
set_selinux_context(homedir)
|
520
|
+
|
521
|
+
notify_observers(:after_initialize_homedir)
|
522
|
+
end
|
523
|
+
|
524
|
+
# Private: Determine next available user id. This is usually determined
|
525
|
+
# and provided by the broker but is auto determined if not
|
526
|
+
# provided.
|
527
|
+
#
|
528
|
+
# Examples:
|
529
|
+
# next_uid =>
|
530
|
+
# # => 504
|
531
|
+
#
|
532
|
+
# Returns Integer value for next available uid.
|
533
|
+
def next_uid
|
534
|
+
uids = IO.readlines("/etc/passwd").map{ |line| line.split(":")[2].to_i }
|
535
|
+
gids = IO.readlines("/etc/group").map{ |line| line.split(":")[2].to_i }
|
536
|
+
min_uid = (@config.get("GEAR_MIN_UID") || "500").to_i
|
537
|
+
max_uid = (@config.get("GEAR_MAX_UID") || "1500").to_i
|
538
|
+
|
539
|
+
(min_uid..max_uid).each do |i|
|
540
|
+
if !uids.include?(i) and !gids.include?(i)
|
541
|
+
return i
|
542
|
+
end
|
543
|
+
end
|
544
|
+
end
|
545
|
+
|
546
|
+
# Private: Initialize OpenShift Port Proxy for this gear
|
547
|
+
#
|
548
|
+
# The port proxy range is determined by configuration and must
|
549
|
+
# produce identical results to the abstract cartridge provided
|
550
|
+
# range.
|
551
|
+
#
|
552
|
+
# Examples:
|
553
|
+
# initialize_openshift_port_proxy
|
554
|
+
# => true
|
555
|
+
# service openshift_port_proxy setproxy 35000 delete 35001 delete etc...
|
556
|
+
#
|
557
|
+
# Returns:
|
558
|
+
# true - port proxy could be initialized properly
|
559
|
+
# false - port proxy could not be initialized properly
|
560
|
+
def initialize_openshift_port_proxy
|
561
|
+
notify_observers(:before_initialize_openshift_port_proxy)
|
562
|
+
|
563
|
+
port_begin = (@config.get("PORT_BEGIN") || "35531").to_i
|
564
|
+
ports_per_user = (@config.get("PORTS_PER_USER") || "5").to_i
|
565
|
+
|
566
|
+
# Note, due to a mismatch between dev and prod this is
|
567
|
+
# intentionally not GEAR_MIN_UID and the range must
|
568
|
+
# wrap back around on itself.
|
569
|
+
uid_begin = (@config.get("UID_BEGIN") || "500").to_i
|
570
|
+
|
571
|
+
wrap_uid = ((65536 - port_begin)/ports_per_user)+uid_begin
|
572
|
+
|
573
|
+
if @uid >= wrap_uid
|
574
|
+
tuid = @uid - wrap_uid + uid_begin
|
575
|
+
else
|
576
|
+
tuid = @uid
|
577
|
+
end
|
578
|
+
|
579
|
+
proxy_port_begin = (tuid-uid_begin) * ports_per_user + port_begin
|
580
|
+
|
581
|
+
proxy_port_range = (proxy_port_begin ... (proxy_port_begin + ports_per_user))
|
582
|
+
|
583
|
+
cmd = %{openshift-port-proxy-cfg setproxy}
|
584
|
+
proxy_port_range.each { |i| cmd << " #{i} delete" }
|
585
|
+
out, err, rc = shellCmd(cmd)
|
586
|
+
Syslog.warning(
|
587
|
+
"WARNING: openshift-port-proxy-cfg failed(#{rc}): #{cmd} stdout: #{out} stderr: #{err}"
|
588
|
+
) unless 0 == rc
|
589
|
+
|
590
|
+
notify_observers(:after_initialize_openshift_port_proxy)
|
591
|
+
return rc == 0
|
592
|
+
end
|
593
|
+
|
594
|
+
|
595
|
+
# Private: Kill all processes for a given gear
|
596
|
+
#
|
597
|
+
# Kill all processes owned by the uid or uuid.
|
598
|
+
# No reason for graceful shutdown first, the directories and user are going
|
599
|
+
# to be removed from the system.
|
600
|
+
#
|
601
|
+
# Examples:
|
602
|
+
# kill_gear_procs
|
603
|
+
# => true
|
604
|
+
# pkill -u id
|
605
|
+
#
|
606
|
+
# Raises exception on error.
|
607
|
+
#
|
608
|
+
def self.kill_procs(id)
|
609
|
+
if id.nil? or id == ""
|
610
|
+
raise ArgumentError, "Supplied ID must be a uid."
|
611
|
+
end
|
612
|
+
|
613
|
+
# Give it a good try to delete all processes.
|
614
|
+
# This abuse is neccessary to release locks on polyinstantiated
|
615
|
+
# directories by pam_namespace.
|
616
|
+
out = err = rc = nil
|
617
|
+
10.times do |i|
|
618
|
+
OpenShift::Utils::ShellExec.shellCmd(%{/usr/bin/pkill -9 -u #{id}})
|
619
|
+
out,err,rc = OpenShift::Utils::ShellExec.shellCmd(%{/usr/bin/pgrep -u #{id}})
|
620
|
+
break unless 0 == rc
|
621
|
+
|
622
|
+
Syslog.alert "ERROR: attempt #{i}/10 there are running \"killed\" processes for #{id}(#{rc}): stdout: #{out} stderr: #{err}"
|
623
|
+
sleep 0.5
|
624
|
+
end
|
625
|
+
|
626
|
+
# looks backwards but 0 implies processes still existed
|
627
|
+
if 0 == rc
|
628
|
+
out,err,rc = OpenShift::Utils::ShellExec.shellCmd("ps -u #{@uid} -o state,pid,ppid,cmd")
|
629
|
+
Syslog.alert "ERROR: failed to kill all processes for #{id}(#{rc}): stdout: #{out} stderr: #{err}"
|
630
|
+
end
|
631
|
+
end
|
632
|
+
|
633
|
+
# Private: Purge IPC entities for a given gear
|
634
|
+
#
|
635
|
+
# Enumerate and remove all IPC entities for a given user ID or
|
636
|
+
# user name.
|
637
|
+
#
|
638
|
+
# Examples:
|
639
|
+
# purge_sysvipc
|
640
|
+
# => true
|
641
|
+
# ipcs -c
|
642
|
+
# ipcrm -s id
|
643
|
+
# ipcrm -m id
|
644
|
+
#
|
645
|
+
# Raises exception on error.
|
646
|
+
#
|
647
|
+
def purge_sysvipc(id)
|
648
|
+
if id.nil? or id == ""
|
649
|
+
raise ArgumentError.new("Supplied ID must be a user name or uid.")
|
650
|
+
end
|
651
|
+
|
652
|
+
['-m', '-q', '-s' ].each do |ipctype|
|
653
|
+
out,err,rc=shellCmd(%{/usr/bin/ipcs -c #{ipctype} 2> /dev/null})
|
654
|
+
out.lines do |ipcl|
|
655
|
+
next unless ipcl=~/^\d/
|
656
|
+
ipcent = ipcl.split
|
657
|
+
if ipcent[2] == id
|
658
|
+
# The ID may already be gone
|
659
|
+
shellCmd(%{/usr/bin/ipcrm #{ipctype} #{ipcent[0]}})
|
660
|
+
end
|
661
|
+
end
|
662
|
+
end
|
663
|
+
end
|
664
|
+
|
665
|
+
# private: Write ssh authorized_keys file
|
666
|
+
#
|
667
|
+
# @param [String] authorized_keys_file ssh authorized_keys path
|
668
|
+
# @param [Hash] keys authorized keys with the comment field as the key
|
669
|
+
# @return [Hash] authorized keys with the comment field as the key
|
670
|
+
def write_ssh_keys(authorized_keys_file, keys)
|
671
|
+
File.open(authorized_keys_file,
|
672
|
+
File::WRONLY|File::TRUNC|File::CREAT,
|
673
|
+
0o0440) do |file|
|
674
|
+
file.write(keys.values.join("\n"))
|
675
|
+
file.write("\n")
|
676
|
+
end
|
677
|
+
FileUtils.chown_R('root', @uuid, authorized_keys_file)
|
678
|
+
set_selinux_context(authorized_keys_file)
|
679
|
+
|
680
|
+
keys
|
681
|
+
end
|
682
|
+
|
683
|
+
# private: Read ssh authorized_keys file
|
684
|
+
#
|
685
|
+
# @param [String] authorized_keys_file ssh authorized_keys path
|
686
|
+
# @return [Hash] authorized keys with the comment field as the key
|
687
|
+
def read_ssh_keys(authorized_keys_file)
|
688
|
+
keys = {}
|
689
|
+
if File.exists? authorized_keys_file
|
690
|
+
File.open(authorized_keys_file, File::RDONLY).each_line do | line |
|
691
|
+
options, key_type, key, comment = line.split
|
692
|
+
keys[comment] = line.chomp
|
693
|
+
end
|
694
|
+
FileUtils.chown_R('root', @uuid, authorized_keys_file)
|
695
|
+
end
|
696
|
+
keys
|
697
|
+
end
|
698
|
+
|
699
|
+
# private: Determine the MCS label for a given uid
|
700
|
+
#
|
701
|
+
# @param [Integer] The user ID
|
702
|
+
# @return [String] The SELinux MCS label
|
703
|
+
def get_mcs_label(uid)
|
704
|
+
if ((uid.to_i < 0) || (uid.to_i>523776))
|
705
|
+
raise ArgumentError, "Supplied UID must be between 0 and 523776."
|
706
|
+
end
|
707
|
+
|
708
|
+
setsize=1023
|
709
|
+
tier=setsize
|
710
|
+
ord=uid.to_i
|
711
|
+
while ord > tier
|
712
|
+
ord -= tier
|
713
|
+
tier -= 1
|
714
|
+
end
|
715
|
+
tier = setsize - tier
|
716
|
+
"s0:c#{tier},c#{ord + tier}"
|
717
|
+
end
|
718
|
+
|
719
|
+
# private: Set the SELinux context on a file or directory
|
720
|
+
#
|
721
|
+
# @param [Integer] The user ID
|
722
|
+
def set_selinux_context(path)
|
723
|
+
mcs_label=get_mcs_label(@uid)
|
724
|
+
|
725
|
+
cmd = "restorecon -R #{path}"
|
726
|
+
out, err, rc = shellCmd(cmd)
|
727
|
+
Syslog.err(
|
728
|
+
"ERROR: unable to restorecon user homedir(#{rc}): #{cmd} stdout: #{out} stderr: #{err}"
|
729
|
+
) unless 0 == rc
|
730
|
+
cmd = "chcon -R -l #{mcs_label} #{path}/*"
|
731
|
+
|
732
|
+
out, err, rc = shellCmd(cmd)
|
733
|
+
Syslog.err(
|
734
|
+
"ERROR: unable to chcon user homedir(#{rc}): #{cmd} stdout: #{out} stderr: #{err}"
|
735
|
+
) unless 0 == rc
|
736
|
+
end
|
737
|
+
end
|
738
|
+
end
|