appsignal 2.9.12.beta.0 → 2.9.18.beta.1
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +5 -5
- data/.travis.yml +19 -16
- data/CHANGELOG.md +38 -2
- data/appsignal.gemspec +4 -3
- data/build_matrix.yml +12 -5
- data/ext/Rakefile +2 -3
- data/ext/agent.yml +40 -37
- data/ext/base.rb +33 -10
- data/ext/extconf.rb +2 -3
- data/gemfiles/que_beta.gemfile +10 -0
- data/gemfiles/rails-6.0.gemfile +1 -1
- data/lib/appsignal/cli/install.rb +34 -10
- data/lib/appsignal/helpers/instrumentation.rb +8 -5
- data/lib/appsignal/helpers/metrics.rb +0 -1
- data/lib/appsignal/hooks/puma.rb +13 -18
- data/lib/appsignal/hooks/sequel.rb +1 -1
- data/lib/appsignal/hooks/sidekiq.rb +21 -7
- data/lib/appsignal/integrations/que.rb +9 -8
- data/lib/appsignal/transaction.rb +5 -0
- data/lib/appsignal/utils/rails_helper.rb +4 -0
- data/lib/appsignal/version.rb +1 -1
- data/lib/puma/plugin/appsignal.rb +26 -0
- data/spec/lib/appsignal/cli/install_spec.rb +51 -7
- data/spec/lib/appsignal/hooks/puma_spec.rb +43 -35
- data/spec/lib/appsignal/hooks/sidekiq_spec.rb +13 -0
- data/spec/lib/appsignal/minutely_spec.rb +11 -48
- data/spec/lib/appsignal/transaction_spec.rb +27 -4
- data/spec/lib/appsignal_spec.rb +13 -5
- data/spec/lib/puma/appsignal_spec.rb +91 -0
- data/spec/support/helpers/wait_for_helper.rb +28 -0
- data/spec/support/mocks/mock_probe.rb +11 -0
- metadata +16 -8
@@ -75,18 +75,39 @@ module Appsignal
|
|
75
75
|
def install_for_rails(config)
|
76
76
|
puts "Installing for Ruby on Rails"
|
77
77
|
|
78
|
-
|
78
|
+
name_overwritten = configure_rails_app_name(config)
|
79
|
+
configure(config, rails_environments, name_overwritten)
|
80
|
+
done_notice
|
81
|
+
end
|
79
82
|
|
80
|
-
|
81
|
-
|
82
|
-
|
83
|
-
|
84
|
-
|
83
|
+
def configure_rails_app_name(config)
|
84
|
+
loaded =
|
85
|
+
begin
|
86
|
+
load Appsignal::Utils::RailsHelper.application_config_path
|
87
|
+
true
|
88
|
+
rescue LoadError, StandardError
|
89
|
+
false
|
90
|
+
end
|
91
|
+
|
92
|
+
name_overwritten = false
|
93
|
+
if loaded
|
94
|
+
config[:name] = Appsignal::Utils::RailsHelper.detected_rails_app_name
|
95
|
+
puts
|
96
|
+
name_overwritten = yes_or_no(
|
97
|
+
" Your app's name is: '#{config[:name]}' \n " \
|
98
|
+
"Do you want to change how this is displayed in AppSignal? " \
|
99
|
+
"(y/n): "
|
100
|
+
)
|
101
|
+
if name_overwritten
|
102
|
+
config[:name] = required_input(" Choose app's display name: ")
|
103
|
+
puts
|
104
|
+
end
|
105
|
+
else
|
106
|
+
puts " Unable to automatically detect your Rails app's name."
|
107
|
+
config[:name] = required_input(" Choose your app's display name for AppSignal.com: ")
|
85
108
|
puts
|
86
109
|
end
|
87
|
-
|
88
|
-
configure(config, rails_environments, name_overwritten)
|
89
|
-
done_notice
|
110
|
+
name_overwritten
|
90
111
|
end
|
91
112
|
|
92
113
|
def install_for_sinatra(config)
|
@@ -227,7 +248,10 @@ module Appsignal
|
|
227
248
|
|
228
249
|
def installed_frameworks
|
229
250
|
[].tap do |out|
|
230
|
-
|
251
|
+
if framework_available?("rails") &&
|
252
|
+
File.exist?(Appsignal::Utils::RailsHelper.application_config_path)
|
253
|
+
out << :rails
|
254
|
+
end
|
231
255
|
out << :sinatra if framework_available? "sinatra"
|
232
256
|
out << :padrino if framework_available? "padrino"
|
233
257
|
out << :grape if framework_available? "grape"
|
@@ -2,7 +2,6 @@
|
|
2
2
|
|
3
3
|
module Appsignal
|
4
4
|
module Helpers
|
5
|
-
# @api private
|
6
5
|
module Instrumentation # rubocop:disable Metrics/ModuleLength
|
7
6
|
# Creates an AppSignal transaction for the given block.
|
8
7
|
#
|
@@ -202,7 +201,8 @@ module Appsignal
|
|
202
201
|
)
|
203
202
|
return unless active?
|
204
203
|
unless error.is_a?(Exception)
|
205
|
-
logger.error
|
204
|
+
logger.error "Appsignal.send_error: Cannot send error. The given " \
|
205
|
+
"value is not an exception: #{error.inspect}"
|
206
206
|
return
|
207
207
|
end
|
208
208
|
transaction = Appsignal::Transaction.new(
|
@@ -257,9 +257,12 @@ module Appsignal
|
|
257
257
|
# Exception handling guide
|
258
258
|
# @since 0.6.6
|
259
259
|
def set_error(exception, tags = nil, namespace = nil)
|
260
|
-
|
261
|
-
|
262
|
-
exception.
|
260
|
+
unless exception.is_a?(Exception)
|
261
|
+
logger.error "Appsignal.set_error: Cannot set error. The given " \
|
262
|
+
"value is not an exception: #{exception.inspect}"
|
263
|
+
return
|
264
|
+
end
|
265
|
+
return if !active? || Appsignal::Transaction.current.nil?
|
263
266
|
transaction = Appsignal::Transaction.current
|
264
267
|
transaction.set_error(exception)
|
265
268
|
transaction.set_tags(tags) if tags
|
data/lib/appsignal/hooks/puma.rb
CHANGED
@@ -11,27 +11,22 @@ module Appsignal
|
|
11
11
|
end
|
12
12
|
|
13
13
|
def install
|
14
|
-
if ::Puma.respond_to?(:stats)
|
14
|
+
if ::Puma.respond_to?(:stats) && !defined?(APPSIGNAL_PUMA_PLUGIN_LOADED)
|
15
|
+
# Only install the minutely probe if a user isn't using our Puma
|
16
|
+
# plugin, which lives in `lib/puma/appsignal.rb`. This plugin defines
|
17
|
+
# the {APPSIGNAL_PUMA_PLUGIN_LOADED} constant.
|
18
|
+
#
|
19
|
+
# We prefer people use the AppSignal Puma plugin. This fallback is
|
20
|
+
# only there when users relied on our *magic* integration.
|
21
|
+
#
|
22
|
+
# Using the Puma plugin, the minutely probe thread will still run in
|
23
|
+
# Puma workers, for other non-Puma probes, but the Puma probe only
|
24
|
+
# runs in the Puma main process.
|
25
|
+
# For more information:
|
26
|
+
# https://docs.appsignal.com/ruby/integrations/puma.html
|
15
27
|
Appsignal::Minutely.probes.register :puma, PumaProbe
|
16
28
|
end
|
17
29
|
|
18
|
-
if ::Puma.respond_to?(:cli_config) && ::Puma.cli_config
|
19
|
-
::Puma.cli_config.options[:before_fork] ||= []
|
20
|
-
::Puma.cli_config.options[:before_fork] << proc do |_id|
|
21
|
-
Appsignal::Minutely.start
|
22
|
-
end
|
23
|
-
|
24
|
-
::Puma.cli_config.options[:before_worker_boot] ||= []
|
25
|
-
::Puma.cli_config.options[:before_worker_boot] << proc do |_id|
|
26
|
-
Appsignal.forked
|
27
|
-
end
|
28
|
-
|
29
|
-
::Puma.cli_config.options[:before_worker_shutdown] ||= []
|
30
|
-
::Puma.cli_config.options[:before_worker_shutdown] << proc do |_id|
|
31
|
-
Appsignal.stop("puma before_worker_shutdown")
|
32
|
-
end
|
33
|
-
end
|
34
|
-
|
35
30
|
return unless defined?(::Puma::Cluster)
|
36
31
|
# For clustered mode with multiple workers
|
37
32
|
::Puma::Cluster.class_eval do
|
@@ -42,7 +42,7 @@ module Appsignal
|
|
42
42
|
|
43
43
|
def install
|
44
44
|
# Register the extension...
|
45
|
-
if ::Sequel::MAJOR >= 4 && ::Sequel::MINOR >= 35
|
45
|
+
if (::Sequel::MAJOR >= 4 && ::Sequel::MINOR >= 35) || ::Sequel::MAJOR >= 5
|
46
46
|
::Sequel::Database.register_extension(
|
47
47
|
:appsignal_integration,
|
48
48
|
Appsignal::Hooks::SequelLogConnectionExtension
|
@@ -38,13 +38,29 @@ module Appsignal
|
|
38
38
|
end
|
39
39
|
|
40
40
|
def call
|
41
|
-
|
41
|
+
track_redis_info
|
42
|
+
track_stats
|
43
|
+
track_queues
|
44
|
+
end
|
45
|
+
|
46
|
+
private
|
47
|
+
|
48
|
+
attr_reader :cache
|
49
|
+
|
50
|
+
def track_redis_info
|
51
|
+
return unless ::Sidekiq.respond_to?(:redis_info)
|
42
52
|
redis_info = ::Sidekiq.redis_info
|
43
|
-
|
44
|
-
gauge "process_count", stats.processes_size
|
53
|
+
|
45
54
|
gauge "connection_count", redis_info.fetch("connected_clients")
|
46
55
|
gauge "memory_usage", redis_info.fetch("used_memory")
|
47
56
|
gauge "memory_usage_rss", redis_info.fetch("used_memory_rss")
|
57
|
+
end
|
58
|
+
|
59
|
+
def track_stats
|
60
|
+
stats = ::Sidekiq::Stats.new
|
61
|
+
|
62
|
+
gauge "worker_count", stats.workers_size
|
63
|
+
gauge "process_count", stats.processes_size
|
48
64
|
gauge_delta :jobs_processed, "job_count", stats.processed,
|
49
65
|
:status => :processed
|
50
66
|
gauge_delta :jobs_failed, "job_count", stats.failed, :status => :failed
|
@@ -52,7 +68,9 @@ module Appsignal
|
|
52
68
|
gauge_delta :jobs_dead, "job_count", stats.dead_size, :status => :died
|
53
69
|
gauge "job_count", stats.scheduled_size, :status => :scheduled
|
54
70
|
gauge "job_count", stats.enqueued, :status => :enqueued
|
71
|
+
end
|
55
72
|
|
73
|
+
def track_queues
|
56
74
|
::Sidekiq::Queue.all.each do |queue|
|
57
75
|
gauge "queue_length", queue.size, :queue => queue.name
|
58
76
|
# Convert latency from seconds to milliseconds
|
@@ -60,10 +78,6 @@ module Appsignal
|
|
60
78
|
end
|
61
79
|
end
|
62
80
|
|
63
|
-
private
|
64
|
-
|
65
|
-
attr_reader :cache
|
66
|
-
|
67
81
|
# Track a gauge metric with the `sidekiq_` prefix
|
68
82
|
def gauge(key, value, tags = {})
|
69
83
|
tags[:hostname] = hostname if hostname
|
@@ -5,16 +5,17 @@ module Appsignal
|
|
5
5
|
module QuePlugin
|
6
6
|
def self.included(base)
|
7
7
|
base.class_eval do
|
8
|
-
def _run_with_appsignal
|
8
|
+
def _run_with_appsignal(*)
|
9
|
+
local_attrs = respond_to?(:que_attrs) ? que_attrs : attrs
|
9
10
|
env = {
|
10
11
|
:metadata => {
|
11
|
-
:id =>
|
12
|
-
:queue =>
|
13
|
-
:run_at =>
|
14
|
-
:priority =>
|
15
|
-
:attempts =>
|
12
|
+
:id => local_attrs[:job_id] || local_attrs[:id],
|
13
|
+
:queue => local_attrs[:queue],
|
14
|
+
:run_at => local_attrs[:run_at].to_s,
|
15
|
+
:priority => local_attrs[:priority],
|
16
|
+
:attempts => local_attrs[:error_count].to_i
|
16
17
|
},
|
17
|
-
:params =>
|
18
|
+
:params => local_attrs[:args]
|
18
19
|
}
|
19
20
|
|
20
21
|
request = Appsignal::Transaction::GenericRequest.new(env)
|
@@ -31,7 +32,7 @@ module Appsignal
|
|
31
32
|
transaction.set_error(error)
|
32
33
|
raise error
|
33
34
|
ensure
|
34
|
-
transaction.set_action "#{
|
35
|
+
transaction.set_action "#{local_attrs[:job_class]}#run"
|
35
36
|
Appsignal::Transaction.complete_current!
|
36
37
|
end
|
37
38
|
end
|
@@ -264,6 +264,11 @@ module Appsignal
|
|
264
264
|
end
|
265
265
|
|
266
266
|
def set_error(error)
|
267
|
+
unless error.is_a?(Exception)
|
268
|
+
Appsignal.logger.error "Appsignal::Transaction#set_error: Cannot set error. " \
|
269
|
+
"The given value is not an exception: #{error.inspect}"
|
270
|
+
return
|
271
|
+
end
|
267
272
|
return unless error
|
268
273
|
return unless Appsignal.active?
|
269
274
|
|
data/lib/appsignal/version.rb
CHANGED
@@ -0,0 +1,26 @@
|
|
1
|
+
APPSIGNAL_PUMA_PLUGIN_LOADED = true
|
2
|
+
|
3
|
+
# AppSignal Puma plugin
|
4
|
+
#
|
5
|
+
# This plugin ensures the minutely probe thread is started with the Puma
|
6
|
+
# minutely probe in the Puma master process.
|
7
|
+
#
|
8
|
+
# The constant {APPSIGNAL_PUMA_PLUGIN_LOADED} is here to mark the Plugin as
|
9
|
+
# loaded by the rest of the AppSignal gem. This ensures that the Puma minutely
|
10
|
+
# probe is not also started in every Puma workers, which was the old behavior.
|
11
|
+
# See {Appsignal::Hooks::PumaHook#install} for more information.
|
12
|
+
#
|
13
|
+
# For even more information:
|
14
|
+
# https://docs.appsignal.com/ruby/integrations/puma.html
|
15
|
+
Puma::Plugin.create do
|
16
|
+
def start(launcher = nil)
|
17
|
+
launcher.events.on_booted do
|
18
|
+
require "appsignal"
|
19
|
+
if ::Puma.respond_to?(:stats)
|
20
|
+
Appsignal::Minutely.probes.register :puma, Appsignal::Hooks::PumaProbe
|
21
|
+
end
|
22
|
+
Appsignal.start
|
23
|
+
Appsignal.start_logger
|
24
|
+
end
|
25
|
+
end
|
26
|
+
end
|
@@ -16,6 +16,10 @@ describe Appsignal::CLI::Install do
|
|
16
16
|
allow(described_class).to receive(:press_any_key)
|
17
17
|
allow(Appsignal::Demo).to receive(:transmit).and_return(true)
|
18
18
|
end
|
19
|
+
after do
|
20
|
+
FileUtils.rm_rf(tmp_dir)
|
21
|
+
FileUtils.mkdir_p(tmp_dir)
|
22
|
+
end
|
19
23
|
around do |example|
|
20
24
|
original_stdin = $stdin
|
21
25
|
$stdin = StringIO.new
|
@@ -157,16 +161,10 @@ describe Appsignal::CLI::Install do
|
|
157
161
|
shared_examples "capistrano install" do
|
158
162
|
let(:capfile) { File.join(tmp_dir, "Capfile") }
|
159
163
|
before do
|
160
|
-
FileUtils.mkdir_p(tmp_dir)
|
161
|
-
|
162
164
|
enter_app_name "foo"
|
163
165
|
add_cli_input "n"
|
164
166
|
choose_environment_config
|
165
167
|
end
|
166
|
-
after do
|
167
|
-
FileUtils.rm_rf(tmp_dir)
|
168
|
-
FileUtils.mkdir_p(tmp_dir)
|
169
|
-
end
|
170
168
|
|
171
169
|
context "without Capfile" do
|
172
170
|
it "does nothing" do
|
@@ -260,7 +258,6 @@ describe Appsignal::CLI::Install do
|
|
260
258
|
FileUtils.touch(File.join(environments_dir, "development.rb"))
|
261
259
|
FileUtils.touch(File.join(environments_dir, "staging.rb"))
|
262
260
|
FileUtils.touch(File.join(environments_dir, "production.rb"))
|
263
|
-
enter_app_name app_name
|
264
261
|
end
|
265
262
|
|
266
263
|
describe "environments" do
|
@@ -410,6 +407,53 @@ describe Appsignal::CLI::Install do
|
|
410
407
|
end
|
411
408
|
end
|
412
409
|
end
|
410
|
+
|
411
|
+
context "when there is no Rails application.rb file" do
|
412
|
+
before do
|
413
|
+
# Do not detect it as another framework for testing
|
414
|
+
allow(described_class).to receive(:framework_available?).and_call_original
|
415
|
+
allow(described_class).to receive(:framework_available?).with("sinatra").and_return(false)
|
416
|
+
|
417
|
+
File.delete(File.join(config_dir, "application.rb"))
|
418
|
+
expect(File.exist?(File.join(config_dir, "application.rb"))).to eql(false)
|
419
|
+
end
|
420
|
+
|
421
|
+
it "fails the installation" do
|
422
|
+
run
|
423
|
+
|
424
|
+
expect(output).to include("We could not detect which framework you are using.")
|
425
|
+
expect(output).to_not include("Installing for Ruby on Rails")
|
426
|
+
expect(output).to include_complete_install
|
427
|
+
|
428
|
+
expect(File.exist?(config_file_path)).to be(false)
|
429
|
+
end
|
430
|
+
end
|
431
|
+
|
432
|
+
context "when failed to load the Rails application.rb file" do
|
433
|
+
before do
|
434
|
+
File.open(File.join(config_dir, "application.rb"), "w") do |file|
|
435
|
+
file.write("I am invalid code")
|
436
|
+
end
|
437
|
+
end
|
438
|
+
|
439
|
+
it "prompts the user to fill in an app name" do
|
440
|
+
enter_app_name app_name
|
441
|
+
choose_config_file
|
442
|
+
run
|
443
|
+
|
444
|
+
expect(output).to include("Installing for Ruby on Rails")
|
445
|
+
expect(output).to include("Unable to automatically detect your Rails app's name.")
|
446
|
+
expect(output).to include("Choose your app's display name for AppSignal.com:")
|
447
|
+
expect(output).to include_file_config
|
448
|
+
expect(output).to include_complete_install
|
449
|
+
|
450
|
+
expect(config_file).to configure_app_name(app_name)
|
451
|
+
expect(config_file).to configure_push_api_key(push_api_key)
|
452
|
+
expect(config_file).to configure_environment("development")
|
453
|
+
expect(config_file).to configure_environment("staging")
|
454
|
+
expect(config_file).to configure_environment("production")
|
455
|
+
end
|
456
|
+
end
|
413
457
|
end
|
414
458
|
end
|
415
459
|
|
@@ -27,6 +27,8 @@ describe Appsignal::Hooks::PumaHook do
|
|
27
27
|
end
|
28
28
|
|
29
29
|
describe "installation" do
|
30
|
+
before { Appsignal::Minutely.probes.clear }
|
31
|
+
|
30
32
|
context "when not clustered mode" do
|
31
33
|
it "does not add AppSignal stop behavior Puma::Cluster" do
|
32
34
|
expect(defined?(::Puma::Cluster)).to be_falsy
|
@@ -34,9 +36,27 @@ describe Appsignal::Hooks::PumaHook do
|
|
34
36
|
Appsignal::Hooks::PumaHook.new.install
|
35
37
|
end
|
36
38
|
|
37
|
-
|
38
|
-
|
39
|
-
|
39
|
+
context "with APPSIGNAL_PUMA_PLUGIN_LOADED defined" do
|
40
|
+
before do
|
41
|
+
# Set in lib/puma/appsignal.rb
|
42
|
+
APPSIGNAL_PUMA_PLUGIN_LOADED = true
|
43
|
+
end
|
44
|
+
after { Object.send :remove_const, :APPSIGNAL_PUMA_PLUGIN_LOADED }
|
45
|
+
|
46
|
+
it "does not add the Puma minutely probe" do
|
47
|
+
Appsignal::Hooks::PumaHook.new.install
|
48
|
+
expect(Appsignal::Minutely.probes[:puma]).to be_nil
|
49
|
+
end
|
50
|
+
end
|
51
|
+
|
52
|
+
context "without APPSIGNAL_PUMA_PLUGIN_LOADED defined" do
|
53
|
+
it "adds the Puma minutely probe" do
|
54
|
+
expect(defined?(APPSIGNAL_PUMA_PLUGIN_LOADED)).to be_nil
|
55
|
+
|
56
|
+
Appsignal::Hooks::PumaHook.new.install
|
57
|
+
probe = Appsignal::Minutely.probes[:puma]
|
58
|
+
expect(probe).to eql(Appsignal::Hooks::PumaProbe)
|
59
|
+
end
|
40
60
|
end
|
41
61
|
end
|
42
62
|
|
@@ -49,11 +69,11 @@ describe Appsignal::Hooks::PumaHook do
|
|
49
69
|
end
|
50
70
|
end
|
51
71
|
end
|
52
|
-
Appsignal::Hooks::PumaHook.new.install
|
53
72
|
end
|
54
73
|
after { Puma.send(:remove_const, :Cluster) }
|
55
74
|
|
56
75
|
it "adds behavior to Puma::Cluster.stop_workers" do
|
76
|
+
Appsignal::Hooks::PumaHook.new.install
|
57
77
|
cluster = Puma::Cluster.new
|
58
78
|
|
59
79
|
expect(cluster.instance_variable_defined?(:@called)).to be_falsy
|
@@ -62,40 +82,28 @@ describe Appsignal::Hooks::PumaHook do
|
|
62
82
|
expect(cluster.instance_variable_get(:@called)).to be(true)
|
63
83
|
end
|
64
84
|
|
65
|
-
|
66
|
-
|
67
|
-
|
68
|
-
|
69
|
-
|
70
|
-
|
71
|
-
|
72
|
-
context "with nil hooks" do
|
73
|
-
before do
|
74
|
-
Puma.cli_config.options.delete(:before_fork)
|
75
|
-
Puma.cli_config.options.delete(:before_worker_boot)
|
76
|
-
Puma.cli_config.options.delete(:before_worker_shutdown)
|
77
|
-
Appsignal::Hooks::PumaHook.new.install
|
78
|
-
end
|
85
|
+
context "with APPSIGNAL_PUMA_PLUGIN_LOADED defined" do
|
86
|
+
before do
|
87
|
+
# Set in lib/puma/appsignal.rb
|
88
|
+
APPSIGNAL_PUMA_PLUGIN_LOADED = true
|
89
|
+
end
|
90
|
+
after { Object.send :remove_const, :APPSIGNAL_PUMA_PLUGIN_LOADED }
|
79
91
|
|
80
|
-
|
81
|
-
|
82
|
-
|
83
|
-
|
84
|
-
|
85
|
-
end
|
92
|
+
it "does not add the Puma minutely probe" do
|
93
|
+
Appsignal::Hooks::PumaHook.new.install
|
94
|
+
expect(Appsignal::Minutely.probes[:puma]).to be_nil
|
95
|
+
end
|
96
|
+
end
|
86
97
|
|
87
|
-
|
88
|
-
|
89
|
-
|
90
|
-
Puma.cli_config.options[:before_worker_boot] = []
|
91
|
-
Puma.cli_config.options[:before_worker_shutdown] = []
|
92
|
-
Appsignal::Hooks::PumaHook.new.install
|
93
|
-
end
|
98
|
+
context "without APPSIGNAL_PUMA_PLUGIN_LOADED defined" do
|
99
|
+
it "adds the Puma minutely probe" do
|
100
|
+
expect(defined?(APPSIGNAL_PUMA_PLUGIN_LOADED)).to be_nil
|
94
101
|
|
95
|
-
|
96
|
-
|
97
|
-
|
98
|
-
|
102
|
+
Appsignal::Hooks::PumaHook.new.install
|
103
|
+
probe = Appsignal::Minutely.probes[:puma]
|
104
|
+
expect(probe).to eql(Appsignal::Hooks::PumaProbe)
|
105
|
+
end
|
106
|
+
end
|
99
107
|
end
|
100
108
|
end
|
101
109
|
end
|