appsignal 0.12.rc.7 → 0.12.rc.8
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/ext/agent.yml +7 -7
- data/lib/appsignal.rb +3 -15
- data/lib/appsignal/hooks.rb +61 -0
- data/lib/appsignal/hooks/celluloid.rb +29 -0
- data/lib/appsignal/hooks/delayed_job.rb +18 -0
- data/lib/appsignal/hooks/net_http.rb +31 -0
- data/lib/appsignal/hooks/passenger.rb +21 -0
- data/lib/appsignal/hooks/puma.rb +20 -0
- data/lib/appsignal/hooks/rake.rb +42 -0
- data/lib/appsignal/hooks/redis.rb +24 -0
- data/lib/appsignal/hooks/resque.rb +28 -0
- data/lib/appsignal/hooks/sequel.rb +38 -0
- data/lib/appsignal/hooks/sidekiq.rb +68 -0
- data/lib/appsignal/hooks/unicorn.rb +39 -0
- data/lib/appsignal/integrations/capistrano/capistrano_2_tasks.rb +27 -29
- data/lib/appsignal/integrations/delayed_job_plugin.rb +39 -0
- data/lib/appsignal/integrations/railtie.rb +41 -0
- data/lib/appsignal/transaction.rb +9 -4
- data/lib/appsignal/version.rb +1 -1
- data/spec/lib/appsignal/{integrations/capistrano2_spec.rb → capistrano2_spec.rb} +1 -1
- data/spec/lib/appsignal/{integrations/capistrano3_spec.rb → capistrano3_spec.rb} +0 -0
- data/spec/lib/appsignal/{integrations → hooks}/celluloid_spec.rb +8 -11
- data/spec/lib/appsignal/{integrations → hooks}/delayed_job_spec.rb +7 -10
- data/spec/lib/appsignal/hooks/net_http_spec.rb +55 -0
- data/spec/lib/appsignal/hooks/passenger_spec.rb +24 -0
- data/spec/lib/appsignal/hooks/puma_spec.rb +50 -0
- data/spec/lib/appsignal/{integrations → hooks}/rake_spec.rb +4 -3
- data/spec/lib/appsignal/hooks/redis_spec.rb +59 -0
- data/spec/lib/appsignal/{integrations → hooks}/resque_spec.rb +5 -11
- data/spec/lib/appsignal/{instrumentations → hooks}/sequel_spec.rb +1 -3
- data/spec/lib/appsignal/{integrations → hooks}/sidekiq_spec.rb +20 -20
- data/spec/lib/appsignal/hooks/unicorn_spec.rb +46 -0
- data/spec/lib/appsignal/hooks_spec.rb +76 -0
- data/spec/lib/appsignal/integrations/{rails_spec.rb → railtie_spec.rb} +0 -0
- data/spec/lib/appsignal/transaction_spec.rb +3 -3
- data/spec/lib/appsignal_spec.rb +0 -33
- metadata +46 -42
- data/lib/appsignal/instrumentations/net_http.rb +0 -17
- data/lib/appsignal/instrumentations/redis.rb +0 -13
- data/lib/appsignal/instrumentations/sequel.rb +0 -31
- data/lib/appsignal/integrations/celluloid.rb +0 -19
- data/lib/appsignal/integrations/delayed_job.rb +0 -44
- data/lib/appsignal/integrations/passenger.rb +0 -11
- data/lib/appsignal/integrations/puma.rb +0 -10
- data/lib/appsignal/integrations/rails.rb +0 -43
- data/lib/appsignal/integrations/rake.rb +0 -30
- data/lib/appsignal/integrations/resque.rb +0 -22
- data/lib/appsignal/integrations/sidekiq.rb +0 -62
- data/lib/appsignal/integrations/unicorn.rb +0 -28
- data/spec/lib/appsignal/instrumentations/net_http_spec.rb +0 -40
- data/spec/lib/appsignal/instrumentations/redis_spec.rb +0 -45
- data/spec/lib/appsignal/integrations/passenger_spec.rb +0 -22
- data/spec/lib/appsignal/integrations/puma_spec.rb +0 -52
- data/spec/lib/appsignal/integrations/unicorn_spec.rb +0 -48
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 638229bbdb0e216279fb2f10390b72bf7f030f8b
|
4
|
+
data.tar.gz: 59785b7e222a6eefb6173c113d3b3bf03b74a3f7
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 70b9b08911109dba74572017b040cfcacb7f8ea0b42f73304a24bba61a5a49a8c48f9065b839cf4d40e02cdb6fa513c220bce54c2b461b8fc49f30076ee13ee2
|
7
|
+
data.tar.gz: 09d494df6ba6f0abd07e2a00ac7f52c56e740e64fe682550d7cd6df45c05736fe99b8f1c70ac66809c54aefa74d5e4fa2ce4f52781b5001edba39d38428b5b6a
|
data/ext/agent.yml
CHANGED
@@ -1,15 +1,15 @@
|
|
1
1
|
---
|
2
|
-
:version:
|
2
|
+
:version: 4bf99f0
|
3
3
|
:triples:
|
4
4
|
x86_64-linux:
|
5
|
-
:checksum:
|
6
|
-
:download_url: https://appsignal-agent-releases.global.ssl.fastly.net/
|
5
|
+
:checksum: 7f02d2f36ff715254d9ec7b99c243ad3a2849ed05c79d0e7e11e4c7140aae576
|
6
|
+
:download_url: https://appsignal-agent-releases.global.ssl.fastly.net/4bf99f0/appsignal-agent-x86_64-linux-static.tar.gz
|
7
7
|
:lib_filename: libappsignal.a
|
8
8
|
i686-linux:
|
9
|
-
:checksum:
|
10
|
-
:download_url: https://appsignal-agent-releases.global.ssl.fastly.net/
|
9
|
+
:checksum: 8103f83f4cf7d9bf22ebafb40216aede836d9d368cdf0f15d0d9644f3d9b4cf7
|
10
|
+
:download_url: https://appsignal-agent-releases.global.ssl.fastly.net/4bf99f0/appsignal-agent-i686-linux-static.tar.gz
|
11
11
|
:lib_filename: libappsignal.a
|
12
12
|
x86_64-darwin:
|
13
|
-
:checksum:
|
14
|
-
:download_url: https://appsignal-agent-releases.global.ssl.fastly.net/
|
13
|
+
:checksum: 22937f34d26ea1dce722b102946a2f3daf3c30755d9f451c6440f580e3dc3eeb
|
14
|
+
:download_url: https://appsignal-agent-releases.global.ssl.fastly.net/4bf99f0/appsignal-agent-x86_64-darwin-static.tar.gz
|
15
15
|
:lib_filename: libappsignal.a
|
data/lib/appsignal.rb
CHANGED
@@ -12,12 +12,6 @@ module Appsignal
|
|
12
12
|
class << self
|
13
13
|
attr_accessor :config, :subscriber, :logger, :agent, :in_memory_log, :extension_loaded
|
14
14
|
|
15
|
-
def load_instrumentations
|
16
|
-
require 'appsignal/instrumentations/net_http' if config[:instrument_net_http]
|
17
|
-
require 'appsignal/instrumentations/redis' if config[:instrument_redis]
|
18
|
-
require 'appsignal/instrumentations/sequel' if config[:instrument_sequel]
|
19
|
-
end
|
20
|
-
|
21
15
|
def extensions
|
22
16
|
@extensions ||= []
|
23
17
|
end
|
@@ -50,7 +44,7 @@ module Appsignal
|
|
50
44
|
logger.info("Starting AppSignal #{Appsignal::VERSION} on #{RUBY_VERSION}/#{RUBY_PLATFORM}")
|
51
45
|
config.write_to_environment
|
52
46
|
Appsignal::Extension.start
|
53
|
-
|
47
|
+
Appsignal::Hooks.load_hooks
|
54
48
|
Appsignal::EventFormatter.initialize_formatters
|
55
49
|
initialize_extensions
|
56
50
|
Appsignal::Extension.install_allocation_event_hook if config[:enable_allocation_tracking]
|
@@ -251,19 +245,13 @@ require 'appsignal/extension'
|
|
251
245
|
require 'appsignal/auth_check'
|
252
246
|
require 'appsignal/config'
|
253
247
|
require 'appsignal/event_formatter'
|
248
|
+
require 'appsignal/hooks'
|
254
249
|
require 'appsignal/marker'
|
255
250
|
require 'appsignal/params_sanitizer'
|
251
|
+
require 'appsignal/integrations/railtie' if defined?(::Rails)
|
256
252
|
require 'appsignal/subscriber'
|
257
253
|
require 'appsignal/transaction'
|
258
254
|
require 'appsignal/version'
|
259
255
|
require 'appsignal/rack/js_exception_catcher'
|
260
256
|
require 'appsignal/js_exception_transaction'
|
261
257
|
require 'appsignal/transmitter'
|
262
|
-
require 'appsignal/integrations/celluloid'
|
263
|
-
require 'appsignal/integrations/delayed_job'
|
264
|
-
require 'appsignal/integrations/passenger'
|
265
|
-
require 'appsignal/integrations/puma'
|
266
|
-
require 'appsignal/integrations/sidekiq'
|
267
|
-
require 'appsignal/integrations/rails'
|
268
|
-
require 'appsignal/integrations/resque'
|
269
|
-
require 'appsignal/integrations/unicorn'
|
@@ -0,0 +1,61 @@
|
|
1
|
+
module Appsignal
|
2
|
+
class Hooks
|
3
|
+
class << self
|
4
|
+
def register(name, hook)
|
5
|
+
hooks[name] = hook
|
6
|
+
end
|
7
|
+
|
8
|
+
def load_hooks
|
9
|
+
hooks.each do |name, hook|
|
10
|
+
hook.try_to_install(name)
|
11
|
+
end
|
12
|
+
end
|
13
|
+
|
14
|
+
def hooks
|
15
|
+
@hooks ||= {}
|
16
|
+
end
|
17
|
+
end
|
18
|
+
|
19
|
+
class Hook
|
20
|
+
def self.register(name, hook=self)
|
21
|
+
Appsignal::Hooks.register(name, hook.new)
|
22
|
+
end
|
23
|
+
|
24
|
+
def try_to_install(name)
|
25
|
+
if dependencies_present? && !installed?
|
26
|
+
Appsignal.logger.info("Installing #{name} hook")
|
27
|
+
begin
|
28
|
+
install
|
29
|
+
@installed = true
|
30
|
+
rescue => ex
|
31
|
+
Appsignal.logger.error("Error while installing #{name} hook: #{ex}")
|
32
|
+
end
|
33
|
+
end
|
34
|
+
end
|
35
|
+
|
36
|
+
def installed?
|
37
|
+
!! @installed
|
38
|
+
end
|
39
|
+
|
40
|
+
def dependencies_present?
|
41
|
+
raise NotImplementedError
|
42
|
+
end
|
43
|
+
|
44
|
+
def install
|
45
|
+
raise NotImplementedError
|
46
|
+
end
|
47
|
+
end
|
48
|
+
end
|
49
|
+
end
|
50
|
+
|
51
|
+
require 'appsignal/hooks/celluloid'
|
52
|
+
require 'appsignal/hooks/delayed_job'
|
53
|
+
require 'appsignal/hooks/net_http'
|
54
|
+
require 'appsignal/hooks/passenger'
|
55
|
+
require 'appsignal/hooks/puma'
|
56
|
+
require 'appsignal/hooks/rake'
|
57
|
+
require 'appsignal/hooks/redis'
|
58
|
+
require 'appsignal/hooks/resque'
|
59
|
+
require 'appsignal/hooks/sequel'
|
60
|
+
require 'appsignal/hooks/sidekiq'
|
61
|
+
require 'appsignal/hooks/unicorn'
|
@@ -0,0 +1,29 @@
|
|
1
|
+
module Appsignal
|
2
|
+
class Hooks
|
3
|
+
class CelluloidHook < Appsignal::Hooks::Hook
|
4
|
+
register :celluloid
|
5
|
+
|
6
|
+
def dependencies_present?
|
7
|
+
defined?(::Celluloid)
|
8
|
+
end
|
9
|
+
|
10
|
+
def install
|
11
|
+
# Some versions of Celluloid have race conditions while exiting
|
12
|
+
# that can result in a dead lock. We stop appsignal before shutting
|
13
|
+
# down Celluloid so we're sure our thread does not aggravate this situation.
|
14
|
+
# This way we also make sure any outstanding transactions get flushed.
|
15
|
+
|
16
|
+
::Celluloid.class_eval do
|
17
|
+
class << self
|
18
|
+
alias shutdown_without_appsignal shutdown
|
19
|
+
|
20
|
+
def shutdown
|
21
|
+
Appsignal.stop
|
22
|
+
shutdown_without_appsignal
|
23
|
+
end
|
24
|
+
end
|
25
|
+
end
|
26
|
+
end
|
27
|
+
end
|
28
|
+
end
|
29
|
+
end
|
@@ -0,0 +1,18 @@
|
|
1
|
+
module Appsignal
|
2
|
+
class Hooks
|
3
|
+
class DelayedJobHook < Appsignal::Hooks::Hook
|
4
|
+
register :delayed_job
|
5
|
+
|
6
|
+
def dependencies_present?
|
7
|
+
defined?(::Delayed::Plugin)
|
8
|
+
end
|
9
|
+
|
10
|
+
def install
|
11
|
+
# The DJ plugin is a subclass of Delayed::Plugin, so we can only
|
12
|
+
# require this code if we're actually installing.
|
13
|
+
require 'appsignal/integrations/delayed_job_plugin'
|
14
|
+
::Delayed::Worker.plugins << Appsignal::Hooks::DelayedJobPlugin
|
15
|
+
end
|
16
|
+
end
|
17
|
+
end
|
18
|
+
end
|
@@ -0,0 +1,31 @@
|
|
1
|
+
require 'net/http'
|
2
|
+
|
3
|
+
module Appsignal
|
4
|
+
class Hooks
|
5
|
+
class NetHttpHook < Appsignal::Hooks::Hook
|
6
|
+
register :net_http
|
7
|
+
|
8
|
+
def dependencies_present?
|
9
|
+
Appsignal.config[:instrument_net_http]
|
10
|
+
end
|
11
|
+
|
12
|
+
def install
|
13
|
+
Net::HTTP.class_eval do
|
14
|
+
alias request_without_appsignal request
|
15
|
+
|
16
|
+
def request(request, body=nil, &block)
|
17
|
+
ActiveSupport::Notifications.instrument(
|
18
|
+
'request.net_http',
|
19
|
+
:protocol => use_ssl? ? 'https' : 'http',
|
20
|
+
:domain => request['host'] || self.address,
|
21
|
+
:path => request.path,
|
22
|
+
:method => request.method
|
23
|
+
) do
|
24
|
+
request_without_appsignal(request, body, &block)
|
25
|
+
end
|
26
|
+
end
|
27
|
+
end
|
28
|
+
end
|
29
|
+
end
|
30
|
+
end
|
31
|
+
end
|
@@ -0,0 +1,21 @@
|
|
1
|
+
module Appsignal
|
2
|
+
class Hooks
|
3
|
+
class PassengerHook < Appsignal::Hooks::Hook
|
4
|
+
register :passenger
|
5
|
+
|
6
|
+
def dependencies_present?
|
7
|
+
defined?(::PhusionPassenger)
|
8
|
+
end
|
9
|
+
|
10
|
+
def install
|
11
|
+
::PhusionPassenger.on_event(:starting_worker_process) do |forked|
|
12
|
+
Appsignal.forked
|
13
|
+
end
|
14
|
+
|
15
|
+
::PhusionPassenger.on_event(:stopping_worker_process) do
|
16
|
+
Appsignal.stop
|
17
|
+
end
|
18
|
+
end
|
19
|
+
end
|
20
|
+
end
|
21
|
+
end
|
@@ -0,0 +1,20 @@
|
|
1
|
+
module Appsignal
|
2
|
+
class Hooks
|
3
|
+
class PumaHook < Appsignal::Hooks::Hook
|
4
|
+
register :puma
|
5
|
+
|
6
|
+
def dependencies_present?
|
7
|
+
defined?(::Puma) &&
|
8
|
+
::Puma.respond_to?(:cli_config) &&
|
9
|
+
::Puma.cli_config
|
10
|
+
end
|
11
|
+
|
12
|
+
def install
|
13
|
+
::Puma.cli_config.options[:before_worker_shutdown] ||= []
|
14
|
+
::Puma.cli_config.options[:before_worker_shutdown] << Proc.new do |id|
|
15
|
+
Appsignal.stop
|
16
|
+
end
|
17
|
+
end
|
18
|
+
end
|
19
|
+
end
|
20
|
+
end
|
@@ -0,0 +1,42 @@
|
|
1
|
+
module Appsignal
|
2
|
+
class Hooks
|
3
|
+
class RakeHook < Appsignal::Hooks::Hook
|
4
|
+
register :rake
|
5
|
+
|
6
|
+
def dependencies_present?
|
7
|
+
defined?(::Rake::Task)
|
8
|
+
end
|
9
|
+
|
10
|
+
def install
|
11
|
+
::Rake::Task.class_eval do
|
12
|
+
alias :invoke_without_appsignal :invoke
|
13
|
+
|
14
|
+
def invoke(*args)
|
15
|
+
if Appsignal.active?
|
16
|
+
invoke_with_appsignal(*args)
|
17
|
+
else
|
18
|
+
invoke_without_appsignal(*args)
|
19
|
+
end
|
20
|
+
end
|
21
|
+
|
22
|
+
def invoke_with_appsignal(*args)
|
23
|
+
invoke_without_appsignal(*args)
|
24
|
+
rescue => error
|
25
|
+
transaction = Appsignal::Transaction.create(
|
26
|
+
SecureRandom.uuid,
|
27
|
+
Appsignal::Transaction::BACKGROUND_JOB,
|
28
|
+
Appsignal::Transaction::GenericRequest.new(
|
29
|
+
:params => args
|
30
|
+
)
|
31
|
+
)
|
32
|
+
transaction.set_action(name)
|
33
|
+
transaction.set_error(error)
|
34
|
+
transaction.complete!
|
35
|
+
Appsignal.stop
|
36
|
+
raise error
|
37
|
+
end
|
38
|
+
end
|
39
|
+
end
|
40
|
+
end
|
41
|
+
end
|
42
|
+
end
|
@@ -0,0 +1,24 @@
|
|
1
|
+
module Appsignal
|
2
|
+
class Hooks
|
3
|
+
class RedisHook < Appsignal::Hooks::Hook
|
4
|
+
register :redis
|
5
|
+
|
6
|
+
def dependencies_present?
|
7
|
+
defined?(::Redis) &&
|
8
|
+
Appsignal.config[:instrument_redis]
|
9
|
+
end
|
10
|
+
|
11
|
+
def install
|
12
|
+
::Redis::Client.class_eval do
|
13
|
+
alias process_without_appsignal process
|
14
|
+
|
15
|
+
def process(commands, &block)
|
16
|
+
ActiveSupport::Notifications.instrument('query.redis') do
|
17
|
+
process_without_appsignal(commands, &block)
|
18
|
+
end
|
19
|
+
end
|
20
|
+
end
|
21
|
+
end
|
22
|
+
end
|
23
|
+
end
|
24
|
+
end
|
@@ -0,0 +1,28 @@
|
|
1
|
+
module Appsignal
|
2
|
+
class Hooks
|
3
|
+
module ResquePlugin
|
4
|
+
def around_perform_resque_plugin(*args)
|
5
|
+
Appsignal.monitor_single_transaction(
|
6
|
+
'perform_job.resque',
|
7
|
+
:class => self.to_s,
|
8
|
+
:method => 'perform'
|
9
|
+
) do
|
10
|
+
yield
|
11
|
+
end
|
12
|
+
end
|
13
|
+
end
|
14
|
+
|
15
|
+
class ResqueHook < Appsignal::Hooks::Hook
|
16
|
+
register :resque
|
17
|
+
|
18
|
+
def dependencies_present?
|
19
|
+
defined?(::Resque)
|
20
|
+
end
|
21
|
+
|
22
|
+
def install
|
23
|
+
# Extend the default job class with AppSignal instrumentation
|
24
|
+
::Resque::Job.send(:extend, Appsignal::Hooks::ResquePlugin)
|
25
|
+
end
|
26
|
+
end
|
27
|
+
end
|
28
|
+
end
|
@@ -0,0 +1,38 @@
|
|
1
|
+
module Appsignal
|
2
|
+
class Hooks
|
3
|
+
module SequelExtension
|
4
|
+
# Add query instrumentation
|
5
|
+
def log_yield(sql, args = nil)
|
6
|
+
|
7
|
+
# We'd like to get full sql queries in the payloads as well. To do
|
8
|
+
# that we need to find out a way to ask Sequel which quoting strategy
|
9
|
+
# is used by the adapter. We can then do something similar to the AR
|
10
|
+
# formatter.
|
11
|
+
|
12
|
+
ActiveSupport::Notifications.instrument('sql.sequel') do
|
13
|
+
yield
|
14
|
+
end
|
15
|
+
end
|
16
|
+
end
|
17
|
+
|
18
|
+
class SequelHook < Appsignal::Hooks::Hook
|
19
|
+
register :sequel
|
20
|
+
|
21
|
+
def dependencies_present?
|
22
|
+
defined?(::Sequel::Database) &&
|
23
|
+
Appsignal.config[:instrument_sequel]
|
24
|
+
end
|
25
|
+
|
26
|
+
def install
|
27
|
+
# Register the extension...
|
28
|
+
::Sequel::Database.register_extension(
|
29
|
+
:appsignal_integration,
|
30
|
+
Appsignal::Hooks::SequelExtension
|
31
|
+
)
|
32
|
+
|
33
|
+
# ... and automatically add it to future instances.
|
34
|
+
::Sequel::Database.extension(:appsignal_integration)
|
35
|
+
end
|
36
|
+
end
|
37
|
+
end
|
38
|
+
end
|
@@ -0,0 +1,68 @@
|
|
1
|
+
module Appsignal
|
2
|
+
class Hooks
|
3
|
+
class SidekiqPlugin
|
4
|
+
def job_keys
|
5
|
+
@job_keys ||= Set.new(%w(
|
6
|
+
class args retried_at failed_at
|
7
|
+
error_message error_class backtrace
|
8
|
+
error_backtrace enqueued_at retry
|
9
|
+
))
|
10
|
+
end
|
11
|
+
|
12
|
+
def call(worker, item, queue)
|
13
|
+
Appsignal.monitor_transaction(
|
14
|
+
'perform_job.sidekiq',
|
15
|
+
:class => item['wrapped'] || item['class'],
|
16
|
+
:method => 'perform',
|
17
|
+
:metadata => formatted_metadata(item),
|
18
|
+
:params => format_args(item['args']),
|
19
|
+
:queue_start => item['enqueued_at']
|
20
|
+
) do
|
21
|
+
yield
|
22
|
+
end
|
23
|
+
end
|
24
|
+
|
25
|
+
def formatted_metadata(item)
|
26
|
+
{}.tap do |hsh|
|
27
|
+
item.each do |key, val|
|
28
|
+
hsh[key] = truncate(string_or_inspect(val)) unless job_keys.include?(key)
|
29
|
+
end
|
30
|
+
end
|
31
|
+
end
|
32
|
+
|
33
|
+
def string_or_inspect(string_or_other)
|
34
|
+
if string_or_other.is_a?(String)
|
35
|
+
string_or_other
|
36
|
+
else
|
37
|
+
string_or_other.inspect
|
38
|
+
end
|
39
|
+
end
|
40
|
+
|
41
|
+
def format_args(args)
|
42
|
+
args.map do |arg|
|
43
|
+
truncate(string_or_inspect(arg))
|
44
|
+
end
|
45
|
+
end
|
46
|
+
|
47
|
+
def truncate(text)
|
48
|
+
text.size > 200 ? "#{text[0...197]}..." : text
|
49
|
+
end
|
50
|
+
end
|
51
|
+
|
52
|
+
class SidekiqHook < Appsignal::Hooks::Hook
|
53
|
+
register :sidekiq
|
54
|
+
|
55
|
+
def dependencies_present?
|
56
|
+
defined?(::Sidekiq)
|
57
|
+
end
|
58
|
+
|
59
|
+
def install
|
60
|
+
::Sidekiq.configure_server do |config|
|
61
|
+
config.server_middleware do |chain|
|
62
|
+
chain.add Appsignal::Hooks::SidekiqPlugin
|
63
|
+
end
|
64
|
+
end
|
65
|
+
end
|
66
|
+
end
|
67
|
+
end
|
68
|
+
end
|