ridley-connectors 2.0.0 → 2.0.1
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/CHANGELOG.md +10 -0
- data/bootstrappers/unix_omnibus.erb +91 -0
- data/bootstrappers/windows_omnibus.erb +113 -0
- data/lib/ridley-connectors.rb +10 -0
- data/lib/ridley-connectors/bootstrap_context.rb +100 -0
- data/lib/ridley-connectors/bootstrap_context/unix.rb +66 -0
- data/lib/ridley-connectors/bootstrap_context/windows.rb +118 -0
- data/lib/ridley-connectors/command_context.rb +76 -0
- data/lib/ridley-connectors/command_context/unix_uninstall.rb +42 -0
- data/lib/ridley-connectors/command_context/windows_uninstall.rb +35 -0
- data/lib/ridley-connectors/errors.rb +29 -0
- data/lib/ridley-connectors/host_commander.rb +26 -9
- data/lib/ridley-connectors/host_connector/ssh.rb +3 -0
- data/lib/ridley-connectors/host_connector/winrm.rb +21 -25
- data/lib/ridley-connectors/version.rb +1 -1
- data/ridley-connectors.gemspec +1 -0
- data/scripts/unix_uninstall_omnibus.erb +31 -0
- data/scripts/windows_uninstall_omnibus.erb +10 -0
- data/spec/spec_helper.rb +3 -0
- data/spec/unit/ridley-connectors/bootstrap_context/unix_spec.rb +101 -0
- data/spec/unit/ridley-connectors/bootstrap_context/windows_spec.rb +116 -0
- data/spec/unit/ridley-connectors/bootstrap_context_spec.rb +52 -0
- data/spec/unit/ridley-connectors/client_spec.rb +1 -1
- data/spec/unit/ridley-connectors/command_context_spec.rb +56 -0
- data/spec/unit/ridley-connectors/host_commander_spec.rb +23 -0
- data/spec/unit/ridley-connectors/host_connector/winrm_spec.rb +31 -23
- metadata +36 -3
@@ -0,0 +1,76 @@
|
|
1
|
+
require 'erubis'
|
2
|
+
|
3
|
+
module Ridley
|
4
|
+
module CommandContext
|
5
|
+
# A base class to provide common functionality between OS specific command contexts. A
|
6
|
+
# command context takes an options hash and binds it against a template file. You can then
|
7
|
+
# retrieve the command to be run on a node by calling {CommandContext::Base#command}.
|
8
|
+
#
|
9
|
+
# @example
|
10
|
+
# my_context = MyCommandContext.new(message: "hello, world!")
|
11
|
+
# my_context.command #=> "echo 'hello, world!'"
|
12
|
+
class Base
|
13
|
+
class << self
|
14
|
+
# Build a command context and immediately run it's command
|
15
|
+
#
|
16
|
+
# @param [Hash] options
|
17
|
+
# an options hash to pass to the new CommandContext
|
18
|
+
def command(options = {})
|
19
|
+
new(options).command
|
20
|
+
end
|
21
|
+
|
22
|
+
# Set or get the path to the template file for the inheriting class
|
23
|
+
#
|
24
|
+
# @param [String] filename
|
25
|
+
# the filename (without extension) of the template file to use to bind
|
26
|
+
# the inheriting command context class to
|
27
|
+
#
|
28
|
+
# @return [Pathname]
|
29
|
+
def template_file(filename = nil)
|
30
|
+
return @template_file if filename.nil?
|
31
|
+
@template_file = Ridley.scripts.join("#{filename}.erb")
|
32
|
+
end
|
33
|
+
end
|
34
|
+
|
35
|
+
# @param [Hash] options
|
36
|
+
def initialize(options = {}); end
|
37
|
+
|
38
|
+
# @return [String]
|
39
|
+
def command
|
40
|
+
template.evaluate(self)
|
41
|
+
end
|
42
|
+
|
43
|
+
private
|
44
|
+
|
45
|
+
# @return [Erubis::Eruby]
|
46
|
+
def template
|
47
|
+
@template ||= Erubis::Eruby.new(IO.read(self.class.template_file).chomp)
|
48
|
+
end
|
49
|
+
end
|
50
|
+
|
51
|
+
# A command context for Unix based OSes
|
52
|
+
class Unix < Base
|
53
|
+
# @return [Boolean]
|
54
|
+
attr_reader :sudo
|
55
|
+
|
56
|
+
# @option options [Boolean] :sudo (true)
|
57
|
+
# bootstrap with sudo (default: true)
|
58
|
+
def initialize(options = {})
|
59
|
+
options = options.reverse_merge(sudo: true)
|
60
|
+
@sudo = options[:sudo]
|
61
|
+
end
|
62
|
+
|
63
|
+
# @return [String]
|
64
|
+
def command
|
65
|
+
sudo ? "sudo #{super}" : super
|
66
|
+
end
|
67
|
+
end
|
68
|
+
|
69
|
+
# A command context for Windows based OSes
|
70
|
+
class Windows < Base; end
|
71
|
+
end
|
72
|
+
end
|
73
|
+
|
74
|
+
Dir["#{File.dirname(__FILE__)}/command_context/*.rb"].sort.each do |path|
|
75
|
+
require_relative "command_context/#{File.basename(path, '.rb')}"
|
76
|
+
end
|
@@ -0,0 +1,42 @@
|
|
1
|
+
module Ridley
|
2
|
+
module CommandContext
|
3
|
+
# Context class for generating an uninstall command for Unix based OSes
|
4
|
+
class UnixUninstall < CommandContext::Unix
|
5
|
+
template_file 'unix_uninstall_omnibus'
|
6
|
+
|
7
|
+
# @return [Boolean]
|
8
|
+
attr_reader :skip_chef
|
9
|
+
|
10
|
+
# @option options [Boolena] :skip_chef (false)
|
11
|
+
# skip removal of the Chef package and the contents of the installation
|
12
|
+
# directory. Setting this to true will only remove any data and configurations
|
13
|
+
# generated by running Chef client.
|
14
|
+
def initialize(options = {})
|
15
|
+
super(options)
|
16
|
+
options = options.reverse_merge(skip_chef: false)
|
17
|
+
@skip_chef = options[:skip_chef]
|
18
|
+
end
|
19
|
+
|
20
|
+
# The path to the Chef configuration directory on the target host
|
21
|
+
#
|
22
|
+
# @return [String]
|
23
|
+
def config_directory
|
24
|
+
"/etc/chef"
|
25
|
+
end
|
26
|
+
|
27
|
+
# The path to the Chef data directory on the target host
|
28
|
+
#
|
29
|
+
# @return [String]
|
30
|
+
def data_directory
|
31
|
+
"/var/chef"
|
32
|
+
end
|
33
|
+
|
34
|
+
# The path to the Omnibus Chef installation on the target host
|
35
|
+
#
|
36
|
+
# @return [String]
|
37
|
+
def install_directory
|
38
|
+
"/opt/chef"
|
39
|
+
end
|
40
|
+
end
|
41
|
+
end
|
42
|
+
end
|
@@ -0,0 +1,35 @@
|
|
1
|
+
module Ridley
|
2
|
+
module CommandContext
|
3
|
+
# Context class for generating an uninstall command for Unix based OSes
|
4
|
+
class WindowsUninstall < CommandContext::Windows
|
5
|
+
template_file 'windows_uninstall_omnibus'
|
6
|
+
|
7
|
+
# @return [Boolean]
|
8
|
+
attr_reader :skip_chef
|
9
|
+
|
10
|
+
# @option options [Boolena] :skip_chef (false)
|
11
|
+
# skip removal of the Chef package and the contents of the installation
|
12
|
+
# directory. Setting this to true will only remove any data and configurations
|
13
|
+
# generated by running Chef client.
|
14
|
+
def initialize(options = {})
|
15
|
+
super(options)
|
16
|
+
options = options.reverse_merge(skip_chef: false)
|
17
|
+
@skip_chef = options[:skip_chef]
|
18
|
+
end
|
19
|
+
|
20
|
+
# The path to the Chef configuration directory on the target host
|
21
|
+
#
|
22
|
+
# @return [String]
|
23
|
+
def config_directory
|
24
|
+
"C:\\chef"
|
25
|
+
end
|
26
|
+
|
27
|
+
# The path to the Omnibus Chef installation on the target host
|
28
|
+
#
|
29
|
+
# @return [String]
|
30
|
+
def install_directory
|
31
|
+
"C:\\opscode\\chef"
|
32
|
+
end
|
33
|
+
end
|
34
|
+
end
|
35
|
+
end
|
@@ -0,0 +1,29 @@
|
|
1
|
+
module Ridley
|
2
|
+
module Errors
|
3
|
+
module ConnectorsError; end
|
4
|
+
|
5
|
+
class HostConnectionError < RidleyError
|
6
|
+
include ConnectorsError
|
7
|
+
end
|
8
|
+
|
9
|
+
class DNSResolvError < HostConnectionError
|
10
|
+
include ConnectorsError
|
11
|
+
end
|
12
|
+
|
13
|
+
class BootstrapError < RidleyError; end
|
14
|
+
class RemoteCommandError < RidleyError; end
|
15
|
+
class RemoteScriptError < RemoteCommandError; end
|
16
|
+
class CommandNotProvided < RemoteCommandError
|
17
|
+
attr_reader :connector_type
|
18
|
+
|
19
|
+
# @params [Symbol] connector_type
|
20
|
+
def initialize(connector_type)
|
21
|
+
@connector_type = connector_type
|
22
|
+
end
|
23
|
+
|
24
|
+
def to_s
|
25
|
+
"No command provided in #{connector_type.inspect}, however the #{connector_type.inspect} connector was selected."
|
26
|
+
end
|
27
|
+
end
|
28
|
+
end
|
29
|
+
end
|
@@ -28,7 +28,8 @@ module Ridley
|
|
28
28
|
|
29
29
|
CONNECTOR_PORT_ERRORS = [
|
30
30
|
Errno::ETIMEDOUT, Timeout::Error, SocketError,
|
31
|
-
Errno::ECONNREFUSED, Errno::EHOSTUNREACH, Errno::EADDRNOTAVAIL
|
31
|
+
Errno::ECONNREFUSED, Errno::EHOSTUNREACH, Errno::EADDRNOTAVAIL,
|
32
|
+
Resolv::ResolvError
|
32
33
|
]
|
33
34
|
|
34
35
|
if Buff::RubyEngine.jruby?
|
@@ -190,7 +191,7 @@ module Ridley
|
|
190
191
|
# @param block [Proc]
|
191
192
|
# an optional block that is yielded the best HostConnector
|
192
193
|
#
|
193
|
-
# @return [HostConnector::SSH, HostConnector::WinRM]
|
194
|
+
# @return [HostConnector::SSH, HostConnector::WinRM, NilClass]
|
194
195
|
def connector_for(host, options = {})
|
195
196
|
options[:ssh] ||= Hash.new
|
196
197
|
options[:winrm] ||= Hash.new
|
@@ -205,20 +206,33 @@ module Ridley
|
|
205
206
|
options.delete(:winrm)
|
206
207
|
ssh
|
207
208
|
else
|
208
|
-
|
209
|
+
nil
|
209
210
|
end
|
210
211
|
end
|
211
212
|
|
212
213
|
private
|
213
214
|
|
215
|
+
# A helper method for sending the provided method to a proper
|
216
|
+
# connector actor.
|
217
|
+
#
|
218
|
+
# @param [Symbol] method
|
219
|
+
# the method to call on the connector actor
|
220
|
+
# @param [String] host
|
221
|
+
# the host to connect to
|
222
|
+
# @param [Array] args
|
223
|
+
# the splatted args passed to the method
|
224
|
+
#
|
225
|
+
# @return [HostConnector::Response]
|
214
226
|
def execute(method, host, *args)
|
215
227
|
options = args.last.is_a?(Hash) ? args.pop : Hash.new
|
216
228
|
|
217
|
-
connector_for(host, options)
|
218
|
-
|
219
|
-
|
220
|
-
|
221
|
-
|
229
|
+
connector = connector_for(host, options)
|
230
|
+
if connector.nil?
|
231
|
+
log.warn { "No connector ports open on '#{host}'" }
|
232
|
+
HostConnector::Response.new(host, stderr: "No connector ports open on '#{host}'")
|
233
|
+
else
|
234
|
+
connector.send(method, host, *args, options)
|
235
|
+
end
|
222
236
|
end
|
223
237
|
|
224
238
|
# Checks to see if the given port is open for TCP connections
|
@@ -242,7 +256,10 @@ module Ridley
|
|
242
256
|
}
|
243
257
|
rescue *CONNECTOR_PORT_ERRORS => ex
|
244
258
|
@retry_count -= 1
|
245
|
-
|
259
|
+
if @retry_count > 0
|
260
|
+
log.info { "Retrying connector_port_open? on '#{host}' #{port} due to: #{ex.class}" }
|
261
|
+
retry
|
262
|
+
end
|
246
263
|
false
|
247
264
|
end
|
248
265
|
end
|
@@ -63,6 +63,9 @@ module Ridley
|
|
63
63
|
rescue Net::SSH::Exception => ex
|
64
64
|
response.exit_code = -1
|
65
65
|
response.stderr = ex.inspect
|
66
|
+
rescue => ex
|
67
|
+
response.exit_code = -1
|
68
|
+
response.stderr = "An unknown error occurred: #{ex.class} - #{ex.message}"
|
66
69
|
end
|
67
70
|
|
68
71
|
case response.exit_code
|
@@ -50,8 +50,13 @@ module Ridley
|
|
50
50
|
|
51
51
|
HostConnector::Response.new(host).tap do |response|
|
52
52
|
begin
|
53
|
-
|
54
|
-
|
53
|
+
if options[:force_batch_file] || command.length > CommandUploader::CHUNK_LIMIT
|
54
|
+
log.debug "Detected a command that was longer than #{CommandUploader::CHUNK_LIMIT} characters. " +
|
55
|
+
"Uploading command as a file to the host."
|
56
|
+
command_uploaders << command_uploader = CommandUploader.new(connection)
|
57
|
+
command_uploader.upload(command)
|
58
|
+
command = command_uploader.command
|
59
|
+
end
|
55
60
|
|
56
61
|
log.info "Running WinRM command: '#{command}' on: '#{host}' as: '#{user}'"
|
57
62
|
|
@@ -72,6 +77,15 @@ module Ridley
|
|
72
77
|
rescue ::WinRM::WinRMHTTPTransportError => ex
|
73
78
|
response.exit_code = :transport_error
|
74
79
|
response.stderr = ex.message
|
80
|
+
rescue SocketError, Errno::EHOSTUNREACH
|
81
|
+
response.exit_code = -1
|
82
|
+
response.stderr = "Host unreachable"
|
83
|
+
rescue Errno::ECONNREFUSED
|
84
|
+
response.exit_code = -1
|
85
|
+
response.stderr = "Connection refused"
|
86
|
+
rescue => ex
|
87
|
+
response.exit_code = -1
|
88
|
+
response.stderr = "An unknown error occurred: #{ex.class} - #{ex.message}"
|
75
89
|
end
|
76
90
|
|
77
91
|
case response.exit_code
|
@@ -79,6 +93,8 @@ module Ridley
|
|
79
93
|
log.info "Successfully ran WinRM command on: '#{host}' as: '#{user}'"
|
80
94
|
when :transport_error
|
81
95
|
log.info "A transport error occured while attempting to run a WinRM command on: '#{host}' as: '#{user}'"
|
96
|
+
when -1
|
97
|
+
log.info "Failed to run WinRM command on: '#{host}' as: '#{user}'"
|
82
98
|
else
|
83
99
|
log.info "Successfully ran WinRM command on: '#{host}' as: '#{user}', but it failed"
|
84
100
|
end
|
@@ -86,29 +102,9 @@ module Ridley
|
|
86
102
|
ensure
|
87
103
|
begin
|
88
104
|
command_uploaders.map(&:cleanup)
|
89
|
-
rescue ::WinRM::WinRMHTTPTransportError => ex
|
90
|
-
log.info "Error cleaning up leftover Powershell scripts on some hosts"
|
91
|
-
|
92
|
-
end
|
93
|
-
|
94
|
-
# Returns the command if it does not break the WinRM command length
|
95
|
-
# limit. Otherwise, we return an execution of the command as a batch file.
|
96
|
-
#
|
97
|
-
# @param command [String]
|
98
|
-
# @param options [Hash]
|
99
|
-
#
|
100
|
-
# @option options [TrueClass, FalseClass] :force_batch_file
|
101
|
-
# Always use a batch file to run the command regardless of command length.
|
102
|
-
#
|
103
|
-
# @return [String]
|
104
|
-
def get_command(command, command_uploader, options = {})
|
105
|
-
if !options[:force_batch_file] && command.length < CommandUploader::CHUNK_LIMIT
|
106
|
-
command
|
107
|
-
else
|
108
|
-
log.debug "Detected a command that was longer than #{CommandUploader::CHUNK_LIMIT} characters. " +
|
109
|
-
"Uploading command as a file to the host."
|
110
|
-
command_uploader.upload(command)
|
111
|
-
command_uploader.command
|
105
|
+
rescue ::WinRM::WinRMHTTPTransportError, SocketError, Errno::EHOSTUNREACH, Errno::ECONNREFUSED => ex
|
106
|
+
log.info "Error cleaning up leftover Powershell scripts on some hosts due to: " +
|
107
|
+
"#{ex.class} - #{ex.message}"
|
112
108
|
end
|
113
109
|
end
|
114
110
|
|
data/ridley-connectors.gemspec
CHANGED
@@ -19,6 +19,7 @@ Gem::Specification.new do |s|
|
|
19
19
|
|
20
20
|
s.add_dependency 'celluloid', '~> 0.16.0.pre'
|
21
21
|
s.add_dependency 'celluloid-io', '~> 0.16.0.pre'
|
22
|
+
s.add_dependency 'erubis'
|
22
23
|
s.add_dependency 'net-ssh'
|
23
24
|
s.add_dependency 'ridley', '~> 3.0'
|
24
25
|
s.add_dependency 'winrm', '~> 1.1.0'
|
@@ -0,0 +1,31 @@
|
|
1
|
+
bash -c '
|
2
|
+
if [ -f "/etc/lsb-release" ]; then
|
3
|
+
platform=$(grep DISTRIB_ID /etc/lsb-release | cut -d "=" -f 2 | tr "[A-Z]" "[a-z]")
|
4
|
+
elif [ -f "/etc/debian_version" ]; then
|
5
|
+
platform="debian"
|
6
|
+
elif [ -f "/etc/redhat-release" ]; then
|
7
|
+
platform="el"
|
8
|
+
elif [ -f "/etc/system-release" ]; then
|
9
|
+
platform=$(sed "s/^\(.\+\) release.\+/\1/" /etc/system-release | tr "[A-Z]" "[a-z]")
|
10
|
+
if [ "$platform" = "amazon linux ami" ]; then
|
11
|
+
platform="el"
|
12
|
+
fi
|
13
|
+
elif [ -f "/etc/SuSE-release" ]; then
|
14
|
+
platform="el"
|
15
|
+
fi
|
16
|
+
|
17
|
+
<% unless skip_chef -%>
|
18
|
+
echo "Un-Installing installed Chef package"
|
19
|
+
case "$platform" in
|
20
|
+
"el") yum remove chef -y ;;
|
21
|
+
"debian") apt-get purge chef -y ;;
|
22
|
+
esac
|
23
|
+
<% end -%>
|
24
|
+
|
25
|
+
rm -Rdf <%= config_directory %>
|
26
|
+
rm -Rdf <%= data_directory %>
|
27
|
+
|
28
|
+
<% unless skip_chef -%>
|
29
|
+
rm -Rdf <%= install_directory %>
|
30
|
+
<% end -%>
|
31
|
+
'
|
@@ -0,0 +1,10 @@
|
|
1
|
+
$productName = "Chef"
|
2
|
+
$installDirectory = "<%= install_directory %>"
|
3
|
+
$configDirectory = "<%= config_directory %>"
|
4
|
+
|
5
|
+
<% unless skip_chef -%>
|
6
|
+
$app = Get-WmiObject -Class Win32_Product | Where-Object { $_.Name -match $productName }
|
7
|
+
If ($app) { $app.Uninstall() }
|
8
|
+
If (Test-Path $installDirectory) { Remove-Item -Recurse -Force $installDirectory }
|
9
|
+
<% end -%>
|
10
|
+
If (Test-Path $configDirectory) { Remove-Item -Recurse -Force $configDirectory }
|
data/spec/spec_helper.rb
CHANGED
@@ -15,6 +15,9 @@ def setup_rspec
|
|
15
15
|
WebMock.disable_net_connect!(allow_localhost: true, net_http_connect_on_start: true)
|
16
16
|
end
|
17
17
|
|
18
|
+
config.filter_run focus: true
|
19
|
+
config.run_all_when_everything_filtered = true
|
20
|
+
|
18
21
|
config.before(:all) { Ridley.logger = Celluloid.logger = nil }
|
19
22
|
|
20
23
|
config.before(:each) do
|
@@ -0,0 +1,101 @@
|
|
1
|
+
require 'spec_helper'
|
2
|
+
|
3
|
+
describe Ridley::BootstrapContext::Unix do
|
4
|
+
let(:options) do
|
5
|
+
{
|
6
|
+
server_url: "https://api.opscode.com/organizations/vialstudios",
|
7
|
+
validator_client: "chef-validator",
|
8
|
+
validator_path: fixtures_path.join("my-fake.pem").to_s,
|
9
|
+
encrypted_data_bag_secret: File.read(fixtures_path.join("my-fake.pem"))
|
10
|
+
}
|
11
|
+
end
|
12
|
+
|
13
|
+
describe "ClassMethods" do
|
14
|
+
subject { described_class }
|
15
|
+
|
16
|
+
describe "::new" do
|
17
|
+
context "when no sudo option is passed through" do
|
18
|
+
it "sets a default value of 'true' to 'sudo'" do
|
19
|
+
options.delete(:sudo)
|
20
|
+
obj = subject.new(options)
|
21
|
+
|
22
|
+
obj.send(:sudo).should be_true
|
23
|
+
end
|
24
|
+
end
|
25
|
+
|
26
|
+
context "when the sudo option is passed through as false" do
|
27
|
+
it "sets the value of sudo to 'false' if provided" do
|
28
|
+
options.merge!(sudo: false)
|
29
|
+
obj = subject.new(options)
|
30
|
+
|
31
|
+
obj.send(:sudo).should be_false
|
32
|
+
end
|
33
|
+
end
|
34
|
+
end
|
35
|
+
end
|
36
|
+
|
37
|
+
subject { described_class.new(options) }
|
38
|
+
|
39
|
+
describe "MixinMethods" do
|
40
|
+
|
41
|
+
describe "#templates_path" do
|
42
|
+
it "returns a pathname" do
|
43
|
+
subject.templates_path.should be_a(Pathname)
|
44
|
+
end
|
45
|
+
end
|
46
|
+
|
47
|
+
describe "#first_boot" do
|
48
|
+
it "returns a string" do
|
49
|
+
subject.first_boot.should be_a(String)
|
50
|
+
end
|
51
|
+
end
|
52
|
+
|
53
|
+
describe "#encrypted_data_bag_secret" do
|
54
|
+
it "returns a string" do
|
55
|
+
subject.encrypted_data_bag_secret.should be_a(String)
|
56
|
+
end
|
57
|
+
end
|
58
|
+
|
59
|
+
describe "#validation_key" do
|
60
|
+
it "returns a string" do
|
61
|
+
subject.validation_key.should be_a(String)
|
62
|
+
end
|
63
|
+
end
|
64
|
+
|
65
|
+
describe "template" do
|
66
|
+
it "returns a string" do
|
67
|
+
subject.template.should be_a(Erubis::Eruby)
|
68
|
+
end
|
69
|
+
end
|
70
|
+
end
|
71
|
+
|
72
|
+
describe "#boot_command" do
|
73
|
+
it "returns a string" do
|
74
|
+
subject.boot_command.should be_a(String)
|
75
|
+
end
|
76
|
+
end
|
77
|
+
|
78
|
+
describe "#chef_run" do
|
79
|
+
it "returns a string" do
|
80
|
+
subject.chef_run.should be_a(String)
|
81
|
+
end
|
82
|
+
end
|
83
|
+
|
84
|
+
describe "#chef_config" do
|
85
|
+
it "returns a string" do
|
86
|
+
subject.chef_config.should be_a(String)
|
87
|
+
end
|
88
|
+
end
|
89
|
+
|
90
|
+
describe "#default_template" do
|
91
|
+
it "returns a string" do
|
92
|
+
subject.default_template.should be_a(String)
|
93
|
+
end
|
94
|
+
end
|
95
|
+
|
96
|
+
describe "#bootstrap_directory" do
|
97
|
+
it "returns a string" do
|
98
|
+
subject.bootstrap_directory.should be_a(String)
|
99
|
+
end
|
100
|
+
end
|
101
|
+
end
|