agentless-catalog-executor 0.9.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +7 -0
- data/.dependency_decisions.yml +118 -0
- data/.gitignore +15 -0
- data/.rspec +3 -0
- data/.rubocop.yml +97 -0
- data/.travis.yml +7 -0
- data/CHANGELOG.md +44 -0
- data/Dockerfile +42 -0
- data/Gemfile +22 -0
- data/LICENSE +201 -0
- data/README.md +23 -0
- data/Rakefile +52 -0
- data/acceptance/.gitignore +10 -0
- data/acceptance/Gemfile +28 -0
- data/acceptance/Rakefile +16 -0
- data/acceptance/config/gem/options.rb +6 -0
- data/acceptance/config/git/options.rb +7 -0
- data/acceptance/config/package/options.rb +6 -0
- data/acceptance/lib/acceptance/ace_command_helper.rb +49 -0
- data/acceptance/lib/acceptance/ace_setup_helper.rb +45 -0
- data/acceptance/setup/common/pre-suite/.gitkeep +0 -0
- data/acceptance/setup/gem/pre-suite/.gitkeep +0 -0
- data/acceptance/setup/git/pre-suite/.gitkeep +0 -0
- data/acceptance/setup/package/pre-suite/.gitkeep +0 -0
- data/acceptance/tests/.gitkeep +0 -0
- data/agentless-catalog-executor.gemspec +38 -0
- data/config/docker.conf +9 -0
- data/config/local.conf +9 -0
- data/config/transport_tasks_config.rb +49 -0
- data/developer-docs/api.md +139 -0
- data/developer-docs/docker.md +170 -0
- data/docker-compose.yml +12 -0
- data/lib/ace/config.rb +73 -0
- data/lib/ace/configurer.rb +17 -0
- data/lib/ace/error.rb +31 -0
- data/lib/ace/fork_util.rb +39 -0
- data/lib/ace/plugin_cache.rb +79 -0
- data/lib/ace/puppet_util.rb +57 -0
- data/lib/ace/schemas/ace-execute_catalog.json +48 -0
- data/lib/ace/schemas/ace-run_task.json +30 -0
- data/lib/ace/schemas/task.json +74 -0
- data/lib/ace/transport_app.rb +252 -0
- data/lib/ace/version.rb +5 -0
- data/lib/puppet/indirector/catalog/certless.rb +83 -0
- metadata +254 -0
@@ -0,0 +1,48 @@
|
|
1
|
+
{
|
2
|
+
"$schema": "http://json-schema.org/draft-04/schema#",
|
3
|
+
"title": "ACE execute_catalog request",
|
4
|
+
"description": "POST /execute_catalog request schema for ACE",
|
5
|
+
"type": "object",
|
6
|
+
"properties": {
|
7
|
+
"target": {
|
8
|
+
"type": "object",
|
9
|
+
"description": "Contains the Transport schema to connect to the remote target",
|
10
|
+
"properties": {
|
11
|
+
"remote-transport": {
|
12
|
+
"type": "string",
|
13
|
+
"description": "The name of the transport being used"
|
14
|
+
}
|
15
|
+
},
|
16
|
+
"additionalProperties": true,
|
17
|
+
"required": ["remote-transport"]
|
18
|
+
},
|
19
|
+
"compiler": {
|
20
|
+
"type": "object",
|
21
|
+
"description": "Contains additional information to compile the catalog",
|
22
|
+
"properties": {
|
23
|
+
"certname": {
|
24
|
+
"type": "string",
|
25
|
+
"description": "The certname of the target"
|
26
|
+
},
|
27
|
+
"environment": {
|
28
|
+
"type": "string",
|
29
|
+
"description": "The name of the environment for which to compile the catalog."
|
30
|
+
},
|
31
|
+
"transaction_uuid": {
|
32
|
+
"type": "string",
|
33
|
+
"description": "The id for tracking the catalog compilation and report submission."
|
34
|
+
},
|
35
|
+
"job_id": {
|
36
|
+
"type": "string",
|
37
|
+
"description": "The id of the orchestrator job that triggered this run."
|
38
|
+
}
|
39
|
+
},
|
40
|
+
"additionalProperties": true,
|
41
|
+
"required": [
|
42
|
+
"certname",
|
43
|
+
"environment"
|
44
|
+
]
|
45
|
+
}
|
46
|
+
},
|
47
|
+
"required": ["target", "compiler"]
|
48
|
+
}
|
@@ -0,0 +1,30 @@
|
|
1
|
+
{
|
2
|
+
"$schema": "http://json-schema.org/draft-04/schema#",
|
3
|
+
"title": "ACE run_task request",
|
4
|
+
"description": "POST /run_task request schema for ACE",
|
5
|
+
"type": "object",
|
6
|
+
"properties": {
|
7
|
+
"target": {
|
8
|
+
"type": "object",
|
9
|
+
"description": "Contains the Transport schema to connect to the remote target",
|
10
|
+
"properties": {
|
11
|
+
"remote-transport": {
|
12
|
+
"type": "string",
|
13
|
+
"description": "The name of the transport being used"
|
14
|
+
},
|
15
|
+
"run-on": {
|
16
|
+
"type": "string",
|
17
|
+
"description": ""
|
18
|
+
}
|
19
|
+
},
|
20
|
+
"additionalProperties": true,
|
21
|
+
"required": ["remote-transport"]
|
22
|
+
},
|
23
|
+
"task": { "$ref": "file:task"},
|
24
|
+
"parameters": {
|
25
|
+
"type": "object",
|
26
|
+
"description": "JSON formatted parameters to be provided to task"
|
27
|
+
}
|
28
|
+
},
|
29
|
+
"required": ["target", "task"]
|
30
|
+
}
|
@@ -0,0 +1,74 @@
|
|
1
|
+
{
|
2
|
+
"id": "file:task",
|
3
|
+
"$schema": "http://json-schema.org/draft-04/schema#",
|
4
|
+
"title": "Task",
|
5
|
+
"description": "Task schema for bolt-server",
|
6
|
+
"type": "object",
|
7
|
+
"properties": {
|
8
|
+
"name": {
|
9
|
+
"type": "string",
|
10
|
+
"description": "Task name"
|
11
|
+
},
|
12
|
+
"metadata": {
|
13
|
+
"type": "object",
|
14
|
+
"description": "The metadata object is optional, and contains metadata about the task being run",
|
15
|
+
"properties": {
|
16
|
+
"description": {
|
17
|
+
"type": "string",
|
18
|
+
"description": "The task description from it's metadata"
|
19
|
+
},
|
20
|
+
"parameters": {
|
21
|
+
"type": "object",
|
22
|
+
"description": "Object whose keys are parameter names, and values are objects",
|
23
|
+
"properties": {
|
24
|
+
"description": {
|
25
|
+
"type": "string",
|
26
|
+
"description": "Parameter description"
|
27
|
+
},
|
28
|
+
"type": {
|
29
|
+
"type": "string",
|
30
|
+
"description": "The type the parameter should accept"
|
31
|
+
},
|
32
|
+
"sensitive": {
|
33
|
+
"description": "Whether the task runner should treat the parameter value as sensitive",
|
34
|
+
"type": "boolean"
|
35
|
+
}
|
36
|
+
}
|
37
|
+
},
|
38
|
+
"input_method": {
|
39
|
+
"type": "string",
|
40
|
+
"enum": ["stdin", "environment", "powershell"],
|
41
|
+
"description": "What input method should be used to pass params to the task"
|
42
|
+
}
|
43
|
+
}
|
44
|
+
},
|
45
|
+
"files": {
|
46
|
+
"type": "array",
|
47
|
+
"description": "Description of task files",
|
48
|
+
"items": {
|
49
|
+
"type": "object",
|
50
|
+
"properties": {
|
51
|
+
"uri": {
|
52
|
+
"type": "object",
|
53
|
+
"description": "Where is the file"
|
54
|
+
},
|
55
|
+
"sha256": {
|
56
|
+
"type": "string",
|
57
|
+
"description": "checksum of file"
|
58
|
+
},
|
59
|
+
"filename": {
|
60
|
+
"type": "string",
|
61
|
+
"description": "Name of file"
|
62
|
+
},
|
63
|
+
"size": {
|
64
|
+
"type": "number",
|
65
|
+
"description": "Size of file"
|
66
|
+
}
|
67
|
+
}
|
68
|
+
},
|
69
|
+
"required": ["filename", "uri", "sha256"]
|
70
|
+
}
|
71
|
+
},
|
72
|
+
"required": ["name", "files"],
|
73
|
+
"additionalProperties": false
|
74
|
+
}
|
@@ -0,0 +1,252 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require 'ace/error'
|
4
|
+
require 'ace/fork_util'
|
5
|
+
require 'ace/puppet_util'
|
6
|
+
require 'ace/configurer'
|
7
|
+
require 'ace/plugin_cache'
|
8
|
+
require 'bolt_server/file_cache'
|
9
|
+
require 'bolt/executor'
|
10
|
+
require 'bolt/inventory'
|
11
|
+
require 'bolt/target'
|
12
|
+
require 'bolt/task/puppet_server'
|
13
|
+
require 'json-schema'
|
14
|
+
require 'json'
|
15
|
+
require 'sinatra'
|
16
|
+
require 'puppet/util/network_device/base'
|
17
|
+
|
18
|
+
module ACE
|
19
|
+
class TransportApp < Sinatra::Base
|
20
|
+
def initialize(config = nil)
|
21
|
+
@config = config
|
22
|
+
@executor = Bolt::Executor.new(0, load_config: false)
|
23
|
+
tasks_cache_dir = File.join(@config['cache-dir'], 'tasks')
|
24
|
+
@file_cache = BoltServer::FileCache.new(@config.data.merge('cache-dir' => tasks_cache_dir)).setup
|
25
|
+
environments_cache_dir = File.join(@config['cache-dir'], 'environments')
|
26
|
+
@plugins = ACE::PluginCache.new(environments_cache_dir).setup
|
27
|
+
|
28
|
+
@schemas = {
|
29
|
+
"run_task" => JSON.parse(File.read(File.join(__dir__, 'schemas', 'ace-run_task.json'))),
|
30
|
+
"execute_catalog" => JSON.parse(File.read(File.join(__dir__, 'schemas', 'ace-execute_catalog.json')))
|
31
|
+
}
|
32
|
+
shared_schema = JSON::Schema.new(JSON.parse(File.read(File.join(__dir__, 'schemas', 'task.json'))),
|
33
|
+
Addressable::URI.parse("file:task"))
|
34
|
+
JSON::Validator.add_schema(shared_schema)
|
35
|
+
|
36
|
+
ACE::PuppetUtil.init_global_settings(config['ssl-ca-cert'],
|
37
|
+
config['ssl-ca-crls'],
|
38
|
+
config['ssl-key'],
|
39
|
+
config['ssl-cert'],
|
40
|
+
config['cache-dir'],
|
41
|
+
URI.parse(config['puppet-server-uri']))
|
42
|
+
|
43
|
+
super(nil)
|
44
|
+
end
|
45
|
+
|
46
|
+
# Initialises the puppet target.
|
47
|
+
# @param certname The certificate name of the target.
|
48
|
+
# @param transport The transport provider of the target.
|
49
|
+
# @param target Target connection hash or legacy connection URI
|
50
|
+
# @return [Puppet device instance] Returns Puppet device instance
|
51
|
+
# @raise [puppetlabs/ace/invalid_param] If nil parameter or no connection detail found
|
52
|
+
# @example Connect to device.
|
53
|
+
# init_puppet_target('test_device.domain.com', 'panos', JSON.parse("target":{
|
54
|
+
# "remote-transport":"panos",
|
55
|
+
# "host":"fw.example.net",
|
56
|
+
# "user":"foo",
|
57
|
+
# "password":"wibble"
|
58
|
+
# }) ) => panos.device
|
59
|
+
def self.init_puppet_target(certname, transport, target)
|
60
|
+
unless target
|
61
|
+
raise ACE::Error.new("There was an error parsing the Puppet target. 'target' not found",
|
62
|
+
'puppetlabs/ace/invalid_param')
|
63
|
+
end
|
64
|
+
unless certname
|
65
|
+
raise ACE::Error.new("There was an error parsing the Puppet compiler details. 'certname' not found",
|
66
|
+
'puppetlabs/ace/invalid_param')
|
67
|
+
end
|
68
|
+
unless transport
|
69
|
+
raise ACE::Error.new("There was an error parsing the Puppet target. 'transport' not found",
|
70
|
+
'puppetlabs/ace/invalid_param')
|
71
|
+
end
|
72
|
+
|
73
|
+
if target['uri']
|
74
|
+
if target['uri'] =~ URI::DEFAULT_PARSER.make_regexp
|
75
|
+
# Correct URL
|
76
|
+
url = target['uri']
|
77
|
+
else
|
78
|
+
raise ACE::Error.new("There was an error parsing the URI of the Puppet target",
|
79
|
+
'puppetlabs/ace/invalid_param')
|
80
|
+
end
|
81
|
+
else
|
82
|
+
url = Hash[target.map { |(k, v)| [k.to_sym, v] }]
|
83
|
+
url.delete(:"remote-transport")
|
84
|
+
end
|
85
|
+
|
86
|
+
device_struct = Struct.new(:provider, :url, :name, :options)
|
87
|
+
# Return device
|
88
|
+
Puppet::Util::NetworkDevice.init(device_struct.new(transport,
|
89
|
+
url,
|
90
|
+
certname,
|
91
|
+
{}))
|
92
|
+
end
|
93
|
+
|
94
|
+
def scrub_stack_trace(result)
|
95
|
+
if result.dig(:result, '_error', 'details', 'stack_trace')
|
96
|
+
result[:result]['_error']['details'].reject! { |k| k == 'stack_trace' }
|
97
|
+
end
|
98
|
+
if result.dig(:result, '_error', 'details', 'backtrace')
|
99
|
+
result[:result]['_error']['details'].reject! { |k| k == 'backtrace' }
|
100
|
+
end
|
101
|
+
result
|
102
|
+
end
|
103
|
+
|
104
|
+
def validate_schema(schema, body)
|
105
|
+
schema_error = JSON::Validator.fully_validate(schema, body)
|
106
|
+
if schema_error.any?
|
107
|
+
ACE::Error.new("There was an error validating the request body.",
|
108
|
+
'puppetlabs/ace/schema-error',
|
109
|
+
schema_error)
|
110
|
+
end
|
111
|
+
end
|
112
|
+
|
113
|
+
# returns a hash of trusted facts that will be used
|
114
|
+
# to request a catalog for the target
|
115
|
+
def self.trusted_facts(certname)
|
116
|
+
# if the certname is a valid FQDN, it will split
|
117
|
+
# it in to the correct hostname.domain format
|
118
|
+
# otherwise hostname will be the certname and domain
|
119
|
+
# will be empty
|
120
|
+
hostname, domain = certname.split('.', 2)
|
121
|
+
trusted_facts = {
|
122
|
+
"authenticated": "remote",
|
123
|
+
"extensions": {},
|
124
|
+
"certname": certname,
|
125
|
+
"hostname": hostname
|
126
|
+
}
|
127
|
+
trusted_facts[:domain] = domain if domain
|
128
|
+
trusted_facts
|
129
|
+
end
|
130
|
+
|
131
|
+
get "/" do
|
132
|
+
200
|
133
|
+
end
|
134
|
+
|
135
|
+
post "/check" do
|
136
|
+
[200, 'OK']
|
137
|
+
end
|
138
|
+
|
139
|
+
# :nocov:
|
140
|
+
if ENV['RACK_ENV'] == 'dev'
|
141
|
+
get '/admin/gc' do
|
142
|
+
GC.start
|
143
|
+
200
|
144
|
+
end
|
145
|
+
end
|
146
|
+
|
147
|
+
get '/admin/gc_stat' do
|
148
|
+
[200, GC.stat.to_json]
|
149
|
+
end
|
150
|
+
# :nocov:
|
151
|
+
|
152
|
+
# run this with "curl -X POST http://0.0.0.0:44633/run_task -d '{}'"
|
153
|
+
post '/run_task' do
|
154
|
+
content_type :json
|
155
|
+
|
156
|
+
begin
|
157
|
+
body = JSON.parse(request.body.read)
|
158
|
+
rescue StandardError => e
|
159
|
+
request_error = {
|
160
|
+
_error: ACE::Error.new(e.message,
|
161
|
+
'puppetlabs/ace/request_exception',
|
162
|
+
class: e.class, backtrace: e.backtrace)
|
163
|
+
}
|
164
|
+
return [400, request_error.to_json]
|
165
|
+
end
|
166
|
+
|
167
|
+
error = validate_schema(@schemas["run_task"], body)
|
168
|
+
return [400, error.to_json] unless error.nil?
|
169
|
+
|
170
|
+
opts = body['target'].merge('protocol' => 'remote')
|
171
|
+
|
172
|
+
# This is a workaround for Bolt due to the way it expects to receive the target info
|
173
|
+
# see: https://github.com/puppetlabs/bolt/pull/915#discussion_r268280535
|
174
|
+
# Systems calling into ACE will need to determine the nodename/certname and pass this as `name`
|
175
|
+
target = [Bolt::Target.new(body['target']['host'] || body['target']['name'], opts)]
|
176
|
+
|
177
|
+
inventory = Bolt::Inventory.new(nil)
|
178
|
+
|
179
|
+
target.first.inventory = inventory
|
180
|
+
|
181
|
+
task = Bolt::Task::PuppetServer.new(body['task'], @file_cache)
|
182
|
+
|
183
|
+
parameters = body['parameters'] || {}
|
184
|
+
|
185
|
+
result = ForkUtil.isolate do
|
186
|
+
# Since this will only be on one node we can just return the first result
|
187
|
+
results = @executor.run_task(target, task, parameters)
|
188
|
+
scrub_stack_trace(results.first.status_hash)
|
189
|
+
end
|
190
|
+
[200, result.to_json]
|
191
|
+
end
|
192
|
+
|
193
|
+
post '/execute_catalog' do
|
194
|
+
content_type :json
|
195
|
+
|
196
|
+
begin
|
197
|
+
body = JSON.parse(request.body.read)
|
198
|
+
rescue StandardError => e
|
199
|
+
request_error = {
|
200
|
+
_error: ACE::Error.new(e.message,
|
201
|
+
'puppetlabs/ace/request_exception',
|
202
|
+
class: e.class, backtrace: e.backtrace)
|
203
|
+
}
|
204
|
+
return [400, request_error.to_json]
|
205
|
+
end
|
206
|
+
|
207
|
+
error = validate_schema(@schemas["execute_catalog"], body)
|
208
|
+
return [400, error.to_json] unless error.nil?
|
209
|
+
|
210
|
+
environment = body['compiler']['environment']
|
211
|
+
certname = body['compiler']['certname']
|
212
|
+
|
213
|
+
# TODO: (needs groomed) - proper error handling, errors within the block can be rescued
|
214
|
+
# and handled correctly, ACE::Errors we can handle similar to the task
|
215
|
+
# workflow, errors within the Configuer and not as pleasant - can have
|
216
|
+
# _some_ control over them, especially around status codes from
|
217
|
+
# the /v4/catalog endpoint
|
218
|
+
@plugins.with_synced_libdir(environment, certname) do
|
219
|
+
ACE::TransportApp.init_puppet_target(certname, body['target']['remote-transport'], body['target'])
|
220
|
+
configurer = ACE::Configurer.new(body['compiler']['transaction_uuid'], body['compiler']['job_id'])
|
221
|
+
configurer.run(transport_name: certname,
|
222
|
+
environment: environment,
|
223
|
+
network_device: true,
|
224
|
+
pluginsync: false,
|
225
|
+
trusted_facts: ACE::TransportApp.trusted_facts(certname))
|
226
|
+
end
|
227
|
+
|
228
|
+
# simulate expected error cases
|
229
|
+
if body['compiler']['certname'] == 'fail.example.net'
|
230
|
+
[200, { _error: {
|
231
|
+
msg: 'catalog compile failed',
|
232
|
+
kind: 'puppetlabs/ace/compile_failed',
|
233
|
+
details: 'upstream api errors go here'
|
234
|
+
} }.to_json]
|
235
|
+
elsif body['compiler']['certname'] == 'credentials.example.net'
|
236
|
+
[200, { _error: {
|
237
|
+
msg: 'target specification invalid',
|
238
|
+
kind: 'puppetlabs/ace/target_spec',
|
239
|
+
details: 'upstream api errors go here'
|
240
|
+
} }.to_json]
|
241
|
+
elsif body['compiler']['certname'] == 'reports.example.net'
|
242
|
+
[200, { _error: {
|
243
|
+
msg: 'report submission failed',
|
244
|
+
kind: 'puppetlabs/ace/reporting_failed',
|
245
|
+
details: 'upstream api errors go here'
|
246
|
+
} }.to_json]
|
247
|
+
else
|
248
|
+
[200, '{}']
|
249
|
+
end
|
250
|
+
end
|
251
|
+
end
|
252
|
+
end
|
data/lib/ace/version.rb
ADDED
@@ -0,0 +1,83 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require 'puppet/resource/catalog'
|
4
|
+
require 'puppet/indirector/rest'
|
5
|
+
|
6
|
+
module Puppet
|
7
|
+
class Resource
|
8
|
+
class Catalog
|
9
|
+
class Certless < Puppet::Indirector::REST
|
10
|
+
desc "Find certless catalogs over HTTP via REST."
|
11
|
+
|
12
|
+
# Override the REST indirector headers for the certless
|
13
|
+
# catalog indirector as the `puppet/pson` header throws of the
|
14
|
+
# request body and wraps it in a "value":{<request>} which
|
15
|
+
# causes a validation error on the v4/catalog endpoint
|
16
|
+
def headers
|
17
|
+
common_headers = {
|
18
|
+
"Content-Type" => 'text/json',
|
19
|
+
"Accept" => 'application/json',
|
20
|
+
Puppet::Network::HTTP::HEADER_PUPPET_VERSION => Puppet.version
|
21
|
+
}
|
22
|
+
|
23
|
+
add_accept_encoding(common_headers)
|
24
|
+
end
|
25
|
+
|
26
|
+
def find(request)
|
27
|
+
uri = "/puppet/v4/catalog"
|
28
|
+
|
29
|
+
body = {
|
30
|
+
"certname": request.key,
|
31
|
+
"persistence": {
|
32
|
+
"facts": true, "catalog": true
|
33
|
+
},
|
34
|
+
"environment": request.environment.name.to_s,
|
35
|
+
"facts": {
|
36
|
+
"values": request.options[:transport_facts]
|
37
|
+
},
|
38
|
+
"trusted_facts": {
|
39
|
+
"values": request.options[:trusted_facts]
|
40
|
+
},
|
41
|
+
"transaction_uuid": request.options[:transaction_uuid],
|
42
|
+
"job_id": request.options[:job_id],
|
43
|
+
"options": {
|
44
|
+
"prefer_requested_environment": true,
|
45
|
+
"capture_logs": false
|
46
|
+
}
|
47
|
+
}
|
48
|
+
|
49
|
+
response = do_request(request) do |req|
|
50
|
+
http_post(req, uri, body.to_json, headers)
|
51
|
+
end
|
52
|
+
|
53
|
+
if is_http_200?(response)
|
54
|
+
content_type, body = parse_response(response)
|
55
|
+
# the response from the `v4/catalog` endpoint is in the format of
|
56
|
+
# {"catalog": {}} whereas the configurer expects it to be a
|
57
|
+
# flatter structurer, so passing it the catalog contents from the body
|
58
|
+
# is suited as the API is unlikely to change for
|
59
|
+
# this release
|
60
|
+
result = deserialize_find(content_type, JSON.parse(body)['catalog'].to_json)
|
61
|
+
result.name = request.key if result.respond_to?(:name=)
|
62
|
+
result
|
63
|
+
|
64
|
+
elsif is_http_404?(response)
|
65
|
+
return nil unless request.options[:fail_on_404]
|
66
|
+
|
67
|
+
# 404 can get special treatment as the indirector API can not produce a meaningful
|
68
|
+
# reason to why something is not found - it may not be the thing the user is
|
69
|
+
# expecting to find that is missing, but something else (like the environment).
|
70
|
+
# While this way of handling the issue is not perfect, there is at least an error
|
71
|
+
# that makes a user aware of the reason for the failure.
|
72
|
+
#
|
73
|
+
_, body = parse_response(response)
|
74
|
+
msg = format(_("Find %<uri>s resulted in 404 with the message: %<body>s"),
|
75
|
+
uri: elide(uri, 100),
|
76
|
+
body: body)
|
77
|
+
raise Puppet::Error, msg
|
78
|
+
end
|
79
|
+
end
|
80
|
+
end
|
81
|
+
end
|
82
|
+
end
|
83
|
+
end
|