topological_inventory-providers-common 1.0.2 → 1.0.7
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.github/workflows/gem-push.yml +49 -0
- data/.gitignore +3 -0
- data/.rubocop.yml +3 -0
- data/.rubocop_cc.yml +4 -0
- data/.rubocop_local.yml +2 -0
- data/.travis.yml +8 -0
- data/CHANGELOG.md +29 -1
- data/Gemfile +0 -3
- data/lib/topological_inventory/providers/common/logging.rb +8 -0
- data/lib/topological_inventory/providers/common/operations/endpoint_client.rb +3 -0
- data/lib/topological_inventory/providers/common/operations/source.rb +191 -0
- data/lib/topological_inventory/providers/common/operations/sources_api_client.rb +15 -6
- data/lib/topological_inventory/providers/common/save_inventory/saver.rb +13 -4
- data/lib/topological_inventory/providers/common/version.rb +1 -1
- data/spec/spec_helper.rb +22 -0
- data/spec/support/inventory_helper.rb +14 -0
- data/spec/support/shared/availability_check.rb +236 -0
- data/spec/topological_inventory/providers/common/collector_spec.rb +171 -0
- data/spec/topological_inventory/providers/common/collectors/inventory_collection_storage_spec.rb +44 -0
- data/spec/topological_inventory/providers/common/collectors/inventory_collection_wrapper_spec.rb +9 -0
- data/spec/topological_inventory/providers/common/collectors_pool_spec.rb +150 -0
- data/spec/topological_inventory/providers/common/logger_spec.rb +38 -0
- data/spec/topological_inventory/providers/common/operations/processor_spec.rb +102 -0
- data/spec/topological_inventory/providers/common/operations/source_spec.rb +5 -0
- data/spec/topological_inventory/providers/common/save_inventory/saver_spec.rb +65 -0
- data/spec/topological_inventory/providers/common_spec.rb +3 -0
- data/topological_inventory-providers-common.gemspec +7 -1
- metadata +104 -3
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 868c8f764cd11d9888ed99698a309cc199fdf92823963f4e5cae71208d506ff7
|
4
|
+
data.tar.gz: 39c352e8b1306bdb87675bf0ccc1fe08f02b0c13555b2466c4c38d7a76979aca
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: c2a8af390114548a89c7dc5bb43f1a6e3784a5a59ebd2ce1c7451c15726015df1353e3dcbc3755d7637d696707b0d6db93e5d4907920a1c4b09d3f6410130a77
|
7
|
+
data.tar.gz: 46e6596954367336de4049f8024da8b5e63e5be4c60636c77f91f22c3e407229b38aca604137907d1f5cd6411201d9c3b57e4ad5bab2cfb749b2c04e9ca8eeab
|
@@ -0,0 +1,49 @@
|
|
1
|
+
name: Release Ruby Gem
|
2
|
+
|
3
|
+
on:
|
4
|
+
push:
|
5
|
+
branches: [ master ]
|
6
|
+
paths:
|
7
|
+
- 'lib/topological_inventory/providers/common/version.rb'
|
8
|
+
|
9
|
+
jobs:
|
10
|
+
build:
|
11
|
+
name: Build + Publish
|
12
|
+
runs-on: ubuntu-latest
|
13
|
+
|
14
|
+
steps:
|
15
|
+
- uses: actions/checkout@v2
|
16
|
+
|
17
|
+
- name: Set up Ruby 2.6
|
18
|
+
uses: actions/setup-ruby@v1
|
19
|
+
with:
|
20
|
+
ruby-version: 2.6.x
|
21
|
+
|
22
|
+
- name: Read the version.rb
|
23
|
+
id: version_file
|
24
|
+
run: |
|
25
|
+
echo ::set-output name=data::$(grep VERSION lib/topological_inventory/providers/common/version.rb | awk {'print $3'} | tr -d '"')
|
26
|
+
|
27
|
+
- name: Echo the gem version
|
28
|
+
run: |
|
29
|
+
echo "v${{ steps.version_file.outputs.data }}"
|
30
|
+
|
31
|
+
- name: Publish to RubyGems
|
32
|
+
run: |
|
33
|
+
mkdir -p $HOME/.gem
|
34
|
+
touch $HOME/.gem/credentials
|
35
|
+
chmod 0600 $HOME/.gem/credentials
|
36
|
+
printf -- "---\n:rubygems_api_key: ${GEM_HOST_API_KEY}\n" > $HOME/.gem/credentials
|
37
|
+
gem build *.gemspec
|
38
|
+
gem push *.gem
|
39
|
+
env:
|
40
|
+
GEM_HOST_API_KEY: ${{secrets.RUBYGEMS_API_KEY}}
|
41
|
+
|
42
|
+
- name: Create Release
|
43
|
+
id: create_release
|
44
|
+
uses: actions/create-release@v1
|
45
|
+
env:
|
46
|
+
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
47
|
+
with:
|
48
|
+
tag_name: "v${{ steps.version_file.outputs.data }}"
|
49
|
+
release_name: "v${{ steps.version_file.outputs.data }}"
|
data/.gitignore
CHANGED
data/.rubocop.yml
ADDED
data/.rubocop_cc.yml
ADDED
data/.rubocop_local.yml
ADDED
data/.travis.yml
CHANGED
@@ -1,4 +1,5 @@
|
|
1
1
|
---
|
2
|
+
dist: xenial
|
2
3
|
sudo: false
|
3
4
|
language: ruby
|
4
5
|
cache: bundler
|
@@ -8,3 +9,10 @@ rvm:
|
|
8
9
|
before_install:
|
9
10
|
- 'echo ''gem: --no-ri --no-rdoc --no-document'' > ~/.gemrc'
|
10
11
|
- gem install bundler
|
12
|
+
before_script:
|
13
|
+
- curl -L https://codeclimate.com/downloads/test-reporter/test-reporter-latest-linux-amd64
|
14
|
+
> ./cc-test-reporter
|
15
|
+
- chmod +x ./cc-test-reporter
|
16
|
+
- "./cc-test-reporter before-build"
|
17
|
+
after_script:
|
18
|
+
- "./cc-test-reporter after-build --exit-code $TRAVIS_TEST_RESULT"
|
data/CHANGELOG.md
CHANGED
@@ -4,6 +4,29 @@ All notable changes to this project will be documented in this file.
|
|
4
4
|
The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
|
5
5
|
and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
|
6
6
|
|
7
|
+
## [1.0.7] - 2020-07-27
|
8
|
+
Update operations/source model for receptor-enabled availability checks #36
|
9
|
+
Add check for Application subresource under a Source during Availability check #40
|
10
|
+
Remove infinite loop in error messages #43
|
11
|
+
|
12
|
+
## [1.0.6] - 2020-07-06
|
13
|
+
Add some error handling if Sources does not have endpoints/authentications for a source #38
|
14
|
+
Specs for Collector #35
|
15
|
+
|
16
|
+
## [1.0.5] - 2020-06-18
|
17
|
+
Change release workflow to do everything manually #32
|
18
|
+
Add specs to released files #33
|
19
|
+
|
20
|
+
## [1.0.4] - 2020-06-18
|
21
|
+
Common availability check operation #25
|
22
|
+
Rubocop and codecoverage #29
|
23
|
+
Add github workflow to release to rubygems automatically #31
|
24
|
+
|
25
|
+
## [1.0.3] - 2020-06-04
|
26
|
+
### Changed
|
27
|
+
|
28
|
+
Bump Sources API client to 3.0 #26
|
29
|
+
|
7
30
|
## [1.0.2] - 2020-05-15
|
8
31
|
### Changed
|
9
32
|
|
@@ -20,7 +43,12 @@ manageiq-loggers to >= 0.4.2 #20
|
|
20
43
|
## [1.0.0] - 2020-03-19
|
21
44
|
### Initial release to rubygems.org
|
22
45
|
|
23
|
-
[Unreleased]: https://github.com/RedHatInsights/topological_inventory-providers-common/compare/v1.0.
|
46
|
+
[Unreleased]: https://github.com/RedHatInsights/topological_inventory-providers-common/compare/v1.0.7...HEAD
|
47
|
+
[1.0.6]: https://github.com/RedHatInsights/topological_inventory-providers-common/compare/v1.0.6...v1.0.7
|
48
|
+
[1.0.6]: https://github.com/RedHatInsights/topological_inventory-providers-common/compare/v1.0.5...v1.0.6
|
49
|
+
[1.0.5]: https://github.com/RedHatInsights/topological_inventory-providers-common/compare/v1.0.4...v1.0.5
|
50
|
+
[1.0.4]: https://github.com/RedHatInsights/topological_inventory-providers-common/compare/v1.0.3...v1.0.4
|
51
|
+
[1.0.3]: https://github.com/RedHatInsights/topological_inventory-providers-common/compare/v1.0.2...v1.0.3
|
24
52
|
[1.0.2]: https://github.com/RedHatInsights/topological_inventory-providers-common/compare/v1.0.1...v1.0.2
|
25
53
|
[1.0.1]: https://github.com/RedHatInsights/topological_inventory-providers-common/compare/v1.0.0...v1.0.1
|
26
54
|
[1.0.0]: https://github.com/RedHatInsights/topological_inventory-providers-common/releases/v1.0.0
|
data/Gemfile
CHANGED
@@ -6,9 +6,6 @@ require File.join(Bundler::Plugin.index.load_paths("bundler-inject")[0], "bundle
|
|
6
6
|
# Specify your gem's dependencies in topological_inventory-providers-common.gemspec
|
7
7
|
gemspec
|
8
8
|
|
9
|
-
gem "sources-api-client", "~> 1.0"
|
10
|
-
gem "topological_inventory-ingress_api-client", "~> 1.0"
|
11
|
-
|
12
9
|
group :development, :test do
|
13
10
|
gem 'rake', '>= 12.3.3'
|
14
11
|
gem 'pry-byebug'
|
@@ -21,6 +21,14 @@ module TopologicalInventory
|
|
21
21
|
msg = "[#{status.to_s.upcase}] Sweeping inactive records, :sweep_scope => #{sweep_scope}, :source_uid => #{source}, :refresh_state_uuid => #{refresh_state_uuid}"
|
22
22
|
info(msg)
|
23
23
|
end
|
24
|
+
|
25
|
+
def availability_check(message, severity = :info)
|
26
|
+
log_with_prefix("Source#availability_check", message, severity)
|
27
|
+
end
|
28
|
+
|
29
|
+
def log_with_prefix(prefix, message, severity)
|
30
|
+
send(severity, "#{prefix} - #{message}") if respond_to?(severity)
|
31
|
+
end
|
24
32
|
end
|
25
33
|
|
26
34
|
class Logger < ManageIQ::Loggers::CloudWatch
|
@@ -46,6 +46,9 @@ module TopologicalInventory
|
|
46
46
|
|
47
47
|
def default_endpoint
|
48
48
|
@default_endpoint ||= sources_api.fetch_default_endpoint(source_id)
|
49
|
+
raise "Sources API: Endpoint not found! (source id: #{source_id})" if @default_endpoint.nil?
|
50
|
+
|
51
|
+
@default_endpoint
|
49
52
|
end
|
50
53
|
|
51
54
|
def authentication
|
@@ -0,0 +1,191 @@
|
|
1
|
+
require "topological_inventory/providers/common/logging"
|
2
|
+
require "active_support/core_ext/numeric/time"
|
3
|
+
require "topological_inventory/providers/common/operations/sources_api_client"
|
4
|
+
|
5
|
+
module TopologicalInventory
|
6
|
+
module Providers
|
7
|
+
module Common
|
8
|
+
module Operations
|
9
|
+
class Source
|
10
|
+
include Logging
|
11
|
+
|
12
|
+
STATUS_AVAILABLE, STATUS_UNAVAILABLE = %w[available unavailable].freeze
|
13
|
+
|
14
|
+
ERROR_MESSAGES = {
|
15
|
+
:authentication_not_found => "Authentication not found in Sources API",
|
16
|
+
:endpoint_or_application_not_found => "Endpoint or Application not found in Sources API",
|
17
|
+
}.freeze
|
18
|
+
|
19
|
+
LAST_CHECKED_AT_THRESHOLD = 5.minutes.freeze
|
20
|
+
AUTH_NOT_NECESSARY = "n/a".freeze
|
21
|
+
|
22
|
+
attr_accessor :params, :request_context, :source_id, :account_number
|
23
|
+
|
24
|
+
def initialize(params = {}, request_context = nil)
|
25
|
+
self.params = params
|
26
|
+
self.request_context = request_context
|
27
|
+
self.source_id = params['source_id']
|
28
|
+
self.account_number = params['external_tenant']
|
29
|
+
end
|
30
|
+
|
31
|
+
def availability_check
|
32
|
+
return if params_missing?
|
33
|
+
|
34
|
+
return if checked_recently?
|
35
|
+
|
36
|
+
status, error_message = connection_status
|
37
|
+
|
38
|
+
update_source_and_subresources(status, error_message)
|
39
|
+
|
40
|
+
logger.availability_check("Completed: Source #{source_id} is #{status}")
|
41
|
+
end
|
42
|
+
|
43
|
+
private
|
44
|
+
|
45
|
+
def required_params
|
46
|
+
%w[source_id]
|
47
|
+
end
|
48
|
+
|
49
|
+
def params_missing?
|
50
|
+
is_missing = false
|
51
|
+
required_params.each do |attr|
|
52
|
+
if (is_missing = params[attr].blank?)
|
53
|
+
logger.availability_check("Missing #{attr} for the availability_check request [Source ID: #{source_id}]", :error)
|
54
|
+
break
|
55
|
+
end
|
56
|
+
end
|
57
|
+
|
58
|
+
is_missing
|
59
|
+
end
|
60
|
+
|
61
|
+
def checked_recently?
|
62
|
+
checked_recently = if endpoint.present?
|
63
|
+
endpoint.last_checked_at.present? && endpoint.last_checked_at >= LAST_CHECKED_AT_THRESHOLD.ago
|
64
|
+
elsif application.present?
|
65
|
+
application.last_checked_at.present? && application.last_checked_at >= LAST_CHECKED_AT_THRESHOLD.ago
|
66
|
+
end
|
67
|
+
|
68
|
+
logger.availability_check("Skipping, last check at #{endpoint.last_checked_at || application.last_checked_at} [Source ID: #{source_id}] ") if checked_recently
|
69
|
+
|
70
|
+
checked_recently
|
71
|
+
end
|
72
|
+
|
73
|
+
def connection_status
|
74
|
+
# we need either an endpoint or application to check the source.
|
75
|
+
return [STATUS_UNAVAILABLE, ERROR_MESSAGES[:endpoint_or_application_not_found]] unless endpoint || application
|
76
|
+
|
77
|
+
check_time
|
78
|
+
if endpoint
|
79
|
+
endpoint_connection_check
|
80
|
+
elsif application
|
81
|
+
application_connection_check
|
82
|
+
end
|
83
|
+
end
|
84
|
+
|
85
|
+
def endpoint_connection_check
|
86
|
+
return [STATUS_UNAVAILABLE, ERROR_MESSAGES[:authentication_not_found]] unless authentication
|
87
|
+
|
88
|
+
# call down into the operations pod implementation of `Source#connection_check`
|
89
|
+
connection_check
|
90
|
+
end
|
91
|
+
|
92
|
+
def application_connection_check
|
93
|
+
case application.availability_status
|
94
|
+
when "available"
|
95
|
+
[STATUS_AVAILABLE, nil]
|
96
|
+
when "unavailable"
|
97
|
+
[STATUS_UNAVAILABLE, "Application id #{application.id} unavailable"]
|
98
|
+
end
|
99
|
+
end
|
100
|
+
|
101
|
+
# @return [Array<String, String|nil] - STATUS_[UN]AVAILABLE, error message
|
102
|
+
def connection_check
|
103
|
+
raise NotImplementedError, "#{__method__} must be implemented in a subclass"
|
104
|
+
end
|
105
|
+
|
106
|
+
def update_source_and_subresources(status, error_message = nil)
|
107
|
+
logger.availability_check("Updating source [#{source_id}] status [#{status}] message [#{error_message}]")
|
108
|
+
|
109
|
+
update_source(status)
|
110
|
+
|
111
|
+
update_endpoint(status, error_message) if endpoint
|
112
|
+
update_application(status) if application
|
113
|
+
end
|
114
|
+
|
115
|
+
def update_source(status)
|
116
|
+
source = ::SourcesApiClient::Source.new
|
117
|
+
source.availability_status = status
|
118
|
+
source.last_checked_at = check_time
|
119
|
+
source.last_available_at = check_time if status == STATUS_AVAILABLE
|
120
|
+
|
121
|
+
api_client.update_source(source_id, source)
|
122
|
+
rescue ::SourcesApiClient::ApiError => e
|
123
|
+
logger.availability_check("Failed to update Source id:#{source_id} - #{e.message}", :error)
|
124
|
+
end
|
125
|
+
|
126
|
+
def update_endpoint(status, error_message)
|
127
|
+
if endpoint.nil?
|
128
|
+
logger.availability_check("Failed to update Endpoint for Source id:#{source_id}. Endpoint not found", :error)
|
129
|
+
return
|
130
|
+
end
|
131
|
+
|
132
|
+
endpoint_update = ::SourcesApiClient::Endpoint.new
|
133
|
+
|
134
|
+
endpoint_update.availability_status = status
|
135
|
+
endpoint_update.availability_status_error = error_message.to_s
|
136
|
+
endpoint_update.last_checked_at = check_time
|
137
|
+
endpoint_update.last_available_at = check_time if status == STATUS_AVAILABLE
|
138
|
+
|
139
|
+
api_client.update_endpoint(endpoint.id, endpoint_update)
|
140
|
+
rescue ::SourcesApiClient::ApiError => e
|
141
|
+
logger.availability_check("Failed to update Endpoint(ID: #{endpoint.id}) - #{e.message}", :error)
|
142
|
+
end
|
143
|
+
|
144
|
+
def update_application(status)
|
145
|
+
application_update = ::SourcesApiClient::Application.new
|
146
|
+
application_update.last_checked_at = check_time
|
147
|
+
application_update.last_available_at = check_time if status == STATUS_AVAILABLE
|
148
|
+
|
149
|
+
api_client.update_application(application.id, application_update)
|
150
|
+
rescue ::SourcesApiClient::ApiError => e
|
151
|
+
logger.availability_check("Failed to update Application id: #{application.id} - #{e.message}", :error)
|
152
|
+
end
|
153
|
+
|
154
|
+
def endpoint
|
155
|
+
@endpoint ||= api_client.fetch_default_endpoint(source_id)
|
156
|
+
rescue e
|
157
|
+
logger.availability_check("Failed to fetch Endpoint for Source #{source_id}: #{e.message}", :error)
|
158
|
+
end
|
159
|
+
|
160
|
+
def authentication
|
161
|
+
@authentication ||= if endpoint.receptor_node.present?
|
162
|
+
AUTH_NOT_NECESSARY
|
163
|
+
else
|
164
|
+
api_client.fetch_authentication(source_id, endpoint)
|
165
|
+
end
|
166
|
+
rescue e
|
167
|
+
logger.availability_check("Failed to fetch Authentication for Source #{source_id}: #{e.message}", :error)
|
168
|
+
end
|
169
|
+
|
170
|
+
def application
|
171
|
+
@application ||= api_client.fetch_application(source_id)
|
172
|
+
rescue e
|
173
|
+
logger.availability_check("Failed to fetch Application for Source #{source_id}: #{e.message}", :error)
|
174
|
+
end
|
175
|
+
|
176
|
+
def check_time
|
177
|
+
@check_time ||= Time.now.utc
|
178
|
+
end
|
179
|
+
|
180
|
+
def identity
|
181
|
+
@identity ||= {"x-rh-identity" => Base64.strict_encode64({"identity" => {"account_number" => account_number, "user" => {"is_org_admin" => true}}}.to_json)}
|
182
|
+
end
|
183
|
+
|
184
|
+
def api_client
|
185
|
+
@api_client ||= TopologicalInventory::Providers::Common::Operations::SourcesApiClient.new(identity)
|
186
|
+
end
|
187
|
+
end
|
188
|
+
end
|
189
|
+
end
|
190
|
+
end
|
191
|
+
end
|
@@ -5,6 +5,8 @@ module TopologicalInventory
|
|
5
5
|
module Common
|
6
6
|
module Operations
|
7
7
|
class SourcesApiClient < ::SourcesApiClient::ApiClient
|
8
|
+
delegate :update_source, :update_endpoint, :update_application, :to => :api
|
9
|
+
|
8
10
|
INTERNAL_API_PATH = '//internal/v1.0'.freeze
|
9
11
|
|
10
12
|
def initialize(identity = nil)
|
@@ -20,21 +22,28 @@ module TopologicalInventory
|
|
20
22
|
|
21
23
|
def fetch_default_endpoint(source_id)
|
22
24
|
endpoints = api.list_source_endpoints(source_id)&.data || []
|
23
|
-
|
24
|
-
|
25
|
-
raise "Sources API: Endpoint not found! (source id: #{source_id})" if endpoint.nil?
|
25
|
+
endpoints.find(&:default)
|
26
|
+
end
|
26
27
|
|
27
|
-
|
28
|
+
def fetch_application(source_id)
|
29
|
+
applications = api.list_source_applications(source_id)&.data || []
|
30
|
+
applications.first
|
28
31
|
end
|
29
32
|
|
30
|
-
def fetch_authentication(source_id, default_endpoint = nil)
|
33
|
+
def fetch_authentication(source_id, default_endpoint = nil, authtype = nil)
|
31
34
|
endpoint = default_endpoint || fetch_default_endpoint(source_id)
|
32
35
|
return if endpoint.nil?
|
33
36
|
|
34
37
|
endpoint_authentications = api.list_endpoint_authentications(endpoint.id.to_s).data || []
|
35
38
|
return if endpoint_authentications.empty?
|
36
39
|
|
37
|
-
auth_id =
|
40
|
+
auth_id = if authtype.nil?
|
41
|
+
endpoint_authentications.first&.id
|
42
|
+
else
|
43
|
+
endpoint_authentications.detect { |a| a.authtype = authtype }&.id
|
44
|
+
end
|
45
|
+
return if auth_id.nil?
|
46
|
+
|
38
47
|
fetch_authentication_with_password(auth_id)
|
39
48
|
end
|
40
49
|
|
@@ -8,13 +8,14 @@ module TopologicalInventory
|
|
8
8
|
# As defined in:
|
9
9
|
# https://github.com/zendesk/ruby-kafka/blob/02f7e2816e1130c5202764c275e36837f57ca4af/lib/kafka/protocol/message.rb#L11-L17
|
10
10
|
# There is at least 112 bytes that are added as a message header, so we need to keep room for that. Lets make
|
11
|
-
# it
|
12
|
-
|
11
|
+
# it 512 bytes, just for sure.
|
12
|
+
KAFKA_PAYLOAD_MAX_BYTES_DEFAULT = 750_000
|
13
|
+
KAFKA_RESERVED_HEADER_SIZE = 512
|
13
14
|
|
14
|
-
def initialize(client:, logger:, max_bytes:
|
15
|
+
def initialize(client:, logger:, max_bytes: KAFKA_PAYLOAD_MAX_BYTES_DEFAULT)
|
15
16
|
@client = client
|
16
17
|
@logger = logger
|
17
|
-
@max_bytes = max_bytes
|
18
|
+
@max_bytes = payload_max_size(max_bytes)
|
18
19
|
end
|
19
20
|
|
20
21
|
attr_reader :client, :logger, :max_bytes
|
@@ -117,6 +118,14 @@ module TopologicalInventory
|
|
117
118
|
new_inventory[:collections] = []
|
118
119
|
new_inventory
|
119
120
|
end
|
121
|
+
|
122
|
+
def payload_max_size(max_bytes)
|
123
|
+
if ENV['KAFKA_PAYLOAD_MAX_BYTES']
|
124
|
+
max_bytes.clamp(5_000, ENV['KAFKA_PAYLOAD_MAX_BYTES'].to_i) - KAFKA_RESERVED_HEADER_SIZE
|
125
|
+
else
|
126
|
+
max_bytes - KAFKA_RESERVED_HEADER_SIZE
|
127
|
+
end
|
128
|
+
end
|
120
129
|
end
|
121
130
|
end
|
122
131
|
end
|