active_worker 0.50.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/.document +5 -0
- data/.rvmrc +1 -0
- data/Gemfile +22 -0
- data/Gemfile.lock +77 -0
- data/LICENSE.txt +20 -0
- data/README.rdoc +19 -0
- data/Rakefile +46 -0
- data/VERSION +1 -0
- data/active_worker.gemspec +108 -0
- data/lib/active_worker.rb +29 -0
- data/lib/active_worker/behavior/acts_as_root_object.rb +80 -0
- data/lib/active_worker/behavior/can_be_notified.rb +23 -0
- data/lib/active_worker/behavior/create_from_error.rb +21 -0
- data/lib/active_worker/behavior/execute_concurrently.rb +142 -0
- data/lib/active_worker/behavior/has_modes.rb +44 -0
- data/lib/active_worker/behavior/has_root_object.rb +50 -0
- data/lib/active_worker/behavior/hashable.rb +79 -0
- data/lib/active_worker/configuration.rb +143 -0
- data/lib/active_worker/controller.rb +112 -0
- data/lib/active_worker/event.rb +68 -0
- data/lib/active_worker/expandable.rb +77 -0
- data/lib/active_worker/failure_event.rb +16 -0
- data/lib/active_worker/finished_event.rb +9 -0
- data/lib/active_worker/host_information.rb +13 -0
- data/lib/active_worker/job_queue/job_executer.rb +52 -0
- data/lib/active_worker/job_queue/queue_manager.rb +46 -0
- data/lib/active_worker/job_queue/run_remotely.rb +52 -0
- data/lib/active_worker/modes_map.rb +37 -0
- data/lib/active_worker/notification_event.rb +9 -0
- data/lib/active_worker/parent_event.rb +5 -0
- data/lib/active_worker/started_event.rb +10 -0
- data/lib/active_worker/templatable.rb +46 -0
- data/lib/active_worker/template.rb +41 -0
- data/lib/active_worker/termination_event.rb +21 -0
- data/test/mongoid.yml +28 -0
- data/test/test_acts_as_root_object.rb +123 -0
- data/test/test_can_be_notified.rb +44 -0
- data/test/test_configuration.rb +281 -0
- data/test/test_controller.rb +205 -0
- data/test/test_event.rb +75 -0
- data/test/test_execute_concurrently.rb +134 -0
- data/test/test_expandable.rb +113 -0
- data/test/test_failure_event.rb +69 -0
- data/test/test_finished_event.rb +35 -0
- data/test/test_has_modes.rb +56 -0
- data/test/test_helper.rb +120 -0
- data/test/test_integration.rb +56 -0
- data/test/test_job_executer.rb +65 -0
- data/test/test_queue_manager.rb +106 -0
- data/test/test_run_remotely.rb +63 -0
- data/test/test_started_event.rb +23 -0
- data/test/test_templatable.rb +45 -0
- data/test/test_template.rb +29 -0
- data/test/test_termination_event.rb +28 -0
- metadata +201 -0
@@ -0,0 +1,69 @@
|
|
1
|
+
require_relative "test_helper"
|
2
|
+
|
3
|
+
module ActiveWorker
|
4
|
+
class FailureEventTest < ActiveSupport::TestCase
|
5
|
+
|
6
|
+
test "can create a failure event from error and basic params" do
|
7
|
+
exception = create_exception
|
8
|
+
|
9
|
+
config = ActiveWorker::Configuration.create
|
10
|
+
|
11
|
+
original_events = FailureEvent.from_error(config, exception)
|
12
|
+
assert_equal 1, original_events.size
|
13
|
+
original_event = original_events.first
|
14
|
+
|
15
|
+
event = FailureEvent.where(configuration_id: config.id).first
|
16
|
+
|
17
|
+
assert_equal original_event, event
|
18
|
+
assert_equal exception.backtrace.join("\n"), event.stack_trace
|
19
|
+
assert_equal "Mocha::Mock", event.error_type
|
20
|
+
end
|
21
|
+
|
22
|
+
test "event name and error message are used in event display" do
|
23
|
+
exception = create_exception
|
24
|
+
config = ActiveWorker::Configuration.create
|
25
|
+
|
26
|
+
original_event = FailureEvent.from_error(config, exception).first
|
27
|
+
|
28
|
+
assert_match /#{config.event_name}/, original_event.message
|
29
|
+
assert_match /#{exception.message}/, original_event.message
|
30
|
+
end
|
31
|
+
|
32
|
+
test "can use failure events as finished events" do
|
33
|
+
exception = create_exception
|
34
|
+
config = ActiveWorker::Configuration.create
|
35
|
+
original_event = FailureEvent.from_error(config,exception).first
|
36
|
+
event = FinishedEvent.where(configuration_id: config.id).first
|
37
|
+
|
38
|
+
assert_equal original_event, event
|
39
|
+
end
|
40
|
+
|
41
|
+
test "expands for threads unless completed" do
|
42
|
+
exception = create_exception
|
43
|
+
|
44
|
+
config = ExpandableConfig.create number_of_threads: 2
|
45
|
+
|
46
|
+
config.expand_for_threads
|
47
|
+
|
48
|
+
events = FailureEvent.from_error(config, exception)
|
49
|
+
|
50
|
+
assert_equal 2, events.size
|
51
|
+
end
|
52
|
+
|
53
|
+
test "expands for threads" do
|
54
|
+
exception = create_exception
|
55
|
+
|
56
|
+
config = ExpandableConfig.create number_of_threads: 2
|
57
|
+
|
58
|
+
config.expand_for_threads
|
59
|
+
config.finished
|
60
|
+
|
61
|
+
events = FailureEvent.from_error(config, exception)
|
62
|
+
|
63
|
+
assert_equal 1, events.size
|
64
|
+
end
|
65
|
+
|
66
|
+
|
67
|
+
|
68
|
+
end
|
69
|
+
end
|
@@ -0,0 +1,35 @@
|
|
1
|
+
require_relative "test_helper"
|
2
|
+
|
3
|
+
module ActiveWorker
|
4
|
+
class FinishedEventTest < ActiveSupport::TestCase
|
5
|
+
|
6
|
+
test "exists_for_configurations scopes to finished events" do
|
7
|
+
config1 = Configuration.create
|
8
|
+
config2 = Configuration.create
|
9
|
+
config3 = Configuration.create
|
10
|
+
|
11
|
+
FinishedEvent.create configuration: config1
|
12
|
+
FinishedEvent.create configuration: config2
|
13
|
+
|
14
|
+
configs = [config1, config2, config3]
|
15
|
+
|
16
|
+
assert_equal false, FinishedEvent.exists_for_configurations?(configs)
|
17
|
+
|
18
|
+
Event.create configuration: config3
|
19
|
+
|
20
|
+
assert_equal false, FinishedEvent.exists_for_configurations?(configs)
|
21
|
+
|
22
|
+
FinishedEvent.create configuration: config3
|
23
|
+
|
24
|
+
assert FinishedEvent.exists_for_configurations?(configs)
|
25
|
+
end
|
26
|
+
|
27
|
+
test "finished message" do
|
28
|
+
configuration = Configuration.create
|
29
|
+
event = FinishedEvent.create(configuration: configuration)
|
30
|
+
|
31
|
+
assert_match /finished/, event.message
|
32
|
+
end
|
33
|
+
|
34
|
+
end
|
35
|
+
end
|
@@ -0,0 +1,56 @@
|
|
1
|
+
require_relative "test_helper"
|
2
|
+
|
3
|
+
module ActiveWorker
|
4
|
+
class Modeable < Configuration
|
5
|
+
include ActiveWorker::Behavior::HasModes
|
6
|
+
|
7
|
+
field :custom_field
|
8
|
+
|
9
|
+
add_mode :first, custom_field: "mode1", other_field: "default"
|
10
|
+
add_mode :second, custom_field: "mode2"
|
11
|
+
end
|
12
|
+
|
13
|
+
class ModeableWithNoModes < Configuration
|
14
|
+
include ActiveWorker::Behavior::HasModes
|
15
|
+
field :custom_field
|
16
|
+
end
|
17
|
+
|
18
|
+
class HasModesTest < ActiveSupport::TestCase
|
19
|
+
test "can specify modes" do
|
20
|
+
mode_config = Modeable.create mode: :first
|
21
|
+
|
22
|
+
assert_equal [:first, :second], Modeable.modes
|
23
|
+
assert_equal :first, mode_config.mode
|
24
|
+
end
|
25
|
+
|
26
|
+
test "sets custom fields with mode map" do
|
27
|
+
mode_config = Modeable.create mode: :first
|
28
|
+
assert_equal "mode1", mode_config.custom_field
|
29
|
+
assert_equal "default", mode_config.other_field
|
30
|
+
end
|
31
|
+
|
32
|
+
test "mode does not override already set field" do
|
33
|
+
mode_config = Modeable.create mode: :first, custom_field: "set"
|
34
|
+
assert_equal "set", mode_config.custom_field
|
35
|
+
end
|
36
|
+
|
37
|
+
test "does not blow up with unsupported mode" do
|
38
|
+
assert_raise ActiveWorker::Behavior::HasModes::ModeNotSupportedException do
|
39
|
+
Modeable.create mode: "foo"
|
40
|
+
end
|
41
|
+
end
|
42
|
+
|
43
|
+
test "allows no mode to be set" do
|
44
|
+
mode_config = Modeable.create custom_field: "set"
|
45
|
+
assert_equal "set", mode_config.custom_field
|
46
|
+
end
|
47
|
+
|
48
|
+
test "allows no modes to be added" do
|
49
|
+
assert_equal [], ModeableWithNoModes.modes
|
50
|
+
|
51
|
+
end
|
52
|
+
|
53
|
+
end
|
54
|
+
|
55
|
+
|
56
|
+
end
|
data/test/test_helper.rb
ADDED
@@ -0,0 +1,120 @@
|
|
1
|
+
require 'rubygems'
|
2
|
+
require 'bundler'
|
3
|
+
begin
|
4
|
+
Bundler.setup(:default, :development)
|
5
|
+
rescue Bundler::BundlerError => e
|
6
|
+
$stderr.puts e.message
|
7
|
+
$stderr.puts "Run `bundle install` to install missing gems"
|
8
|
+
exit e.status_code
|
9
|
+
end
|
10
|
+
|
11
|
+
require 'active_support/test_case'
|
12
|
+
require 'test/unit'
|
13
|
+
|
14
|
+
$LOAD_PATH.unshift(File.dirname(__FILE__))
|
15
|
+
$LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
|
16
|
+
require 'active_worker'
|
17
|
+
|
18
|
+
require 'mongoid'
|
19
|
+
ENV["MONGOID_ENV"]="test"
|
20
|
+
Mongoid.load!("#{File.dirname(__FILE__)}/mongoid.yml")
|
21
|
+
|
22
|
+
|
23
|
+
Resque.redis.namespace = "resque:active_worker_test"
|
24
|
+
|
25
|
+
|
26
|
+
module ActiveWorker
|
27
|
+
|
28
|
+
class Rootable
|
29
|
+
include Mongoid::Document
|
30
|
+
include ActiveWorker::Behavior::ActsAsRootObject
|
31
|
+
end
|
32
|
+
|
33
|
+
|
34
|
+
class TopConfig < Configuration
|
35
|
+
config_field :top_field
|
36
|
+
|
37
|
+
def child_configs
|
38
|
+
ChildConfig.mine(self).all
|
39
|
+
end
|
40
|
+
end
|
41
|
+
|
42
|
+
class ChildConfig < Configuration
|
43
|
+
config_field :child_field
|
44
|
+
end
|
45
|
+
|
46
|
+
class TemplatableTopConfig < Configuration
|
47
|
+
include Templatable
|
48
|
+
template_field :top_field
|
49
|
+
config_field :other_top_field
|
50
|
+
end
|
51
|
+
|
52
|
+
class TemplatableChildConfig < Configuration
|
53
|
+
include Templatable
|
54
|
+
template_field :child_field
|
55
|
+
config_field :other_child_field
|
56
|
+
end
|
57
|
+
|
58
|
+
class AfterLaunchConfig < ActiveWorker::Configuration
|
59
|
+
after_launch :after_launch_method
|
60
|
+
end
|
61
|
+
|
62
|
+
class ExpandableConfig < Configuration
|
63
|
+
include Expandable
|
64
|
+
include Templatable
|
65
|
+
field :foo
|
66
|
+
template_field :name
|
67
|
+
config_field :size
|
68
|
+
end
|
69
|
+
|
70
|
+
class MappedExpandableConfig < ExpandableConfig
|
71
|
+
def expansion_maps_for(number_of_configurations)
|
72
|
+
maps = []
|
73
|
+
number_of_configurations.times do |value|
|
74
|
+
maps << {size: value}
|
75
|
+
end
|
76
|
+
maps
|
77
|
+
end
|
78
|
+
end
|
79
|
+
|
80
|
+
end
|
81
|
+
|
82
|
+
|
83
|
+
class ActiveSupport::TestCase
|
84
|
+
|
85
|
+
ActiveWorker::JobQueue::RunRemotely.worker_mode = ActiveWorker::JobQueue::RunRemotely::THREADED
|
86
|
+
|
87
|
+
def create_exception
|
88
|
+
error_message = "Error message"
|
89
|
+
error_backtrace = ["line 1", "line 2"]
|
90
|
+
|
91
|
+
error = mock
|
92
|
+
error.stubs(:message).returns(error_message)
|
93
|
+
error.stubs(:backtrace).returns(error_backtrace)
|
94
|
+
error
|
95
|
+
end
|
96
|
+
|
97
|
+
def wait_for_all_configurations
|
98
|
+
ActiveWorker::Configuration.all.each(&:wait_until_completed)
|
99
|
+
end
|
100
|
+
|
101
|
+
def assert_no_failures
|
102
|
+
assert_equal 0, ActiveWorker::FailureEvent.count, "Failures: #{ActiveWorker::FailureEvent.all.map { |e| e.message + "\n" + e.stack_trace }.join}"
|
103
|
+
end
|
104
|
+
|
105
|
+
setup :clear_database
|
106
|
+
teardown :reset_default_worker_mode
|
107
|
+
|
108
|
+
def reset_default_worker_mode
|
109
|
+
ActiveWorker::Controller::local_worker_mode = ActiveWorker::DEFAULT_MODE
|
110
|
+
end
|
111
|
+
|
112
|
+
private
|
113
|
+
|
114
|
+
def clear_database
|
115
|
+
Mongoid.default_session.collections.select { |c| c.name != 'system.indexes' }.each(&:drop)
|
116
|
+
end
|
117
|
+
|
118
|
+
end
|
119
|
+
|
120
|
+
ActiveWorker::JobQueue::JobExecuter.stubs(:log_error)
|
@@ -0,0 +1,56 @@
|
|
1
|
+
require_relative "test_helper"
|
2
|
+
|
3
|
+
module ActiveWorker
|
4
|
+
class IntegrationTest < ActiveSupport::TestCase
|
5
|
+
|
6
|
+
test "can create correct templates" do
|
7
|
+
|
8
|
+
parent_config = TemplatableTopConfig.create(top_field: "top field", other_top_field: "other top field")
|
9
|
+
child_config0 = TemplatableChildConfig.create(child_field: "child field", other_child_field: "other child field", parent_configuration: parent_config)
|
10
|
+
child_config1 = TemplatableChildConfig.create(child_field: "different child field",other_child_field: "different other child field", parent_configuration: parent_config)
|
11
|
+
|
12
|
+
top_template = parent_config.find_template
|
13
|
+
|
14
|
+
assert_kind_of Template, top_template
|
15
|
+
assert_equal TemplatableTopConfig.name, top_template.configuration_type
|
16
|
+
assert_equal 2, top_template.child_templates.count
|
17
|
+
|
18
|
+
assert_equal parent_config.top_field,top_template[:top_field]
|
19
|
+
assert_nil top_template[:other_top_field]
|
20
|
+
|
21
|
+
child_template0 = top_template.child_templates[0]
|
22
|
+
child_template1 = top_template.child_templates[1]
|
23
|
+
|
24
|
+
assert_equal child_config0.child_field, child_template0[:child_field]
|
25
|
+
assert_equal child_config1.child_field, child_template1[:child_field]
|
26
|
+
|
27
|
+
assert_equal TemplatableChildConfig.name, child_template0.configuration_type
|
28
|
+
assert_equal TemplatableChildConfig.name, child_template0.configuration_type
|
29
|
+
|
30
|
+
end
|
31
|
+
|
32
|
+
test "can recreate correct configurations from templates" do
|
33
|
+
parent_config = TemplatableTopConfig.create(top_field: "top field", other_top_field: "other top field")
|
34
|
+
child_config0 = TemplatableChildConfig.create(child_field: "child field", other_child_field: "other child field", parent_configuration: parent_config)
|
35
|
+
child_config1 = TemplatableChildConfig.create(child_field: "different child field",other_child_field: "different other child field", parent_configuration: parent_config)
|
36
|
+
|
37
|
+
top_template = parent_config.find_template
|
38
|
+
|
39
|
+
created_config = top_template.build_configuration
|
40
|
+
|
41
|
+
assert_kind_of TemplatableTopConfig, created_config
|
42
|
+
assert_equal parent_config.top_field, created_config.top_field
|
43
|
+
assert_nil created_config.other_top_field
|
44
|
+
|
45
|
+
created_child_config0 = created_config.configurations[0]
|
46
|
+
created_child_config1 = created_config.configurations[1]
|
47
|
+
|
48
|
+
assert_equal child_config0.child_field, created_child_config0.child_field
|
49
|
+
assert_equal child_config1.child_field, created_child_config1.child_field
|
50
|
+
|
51
|
+
assert_nil created_child_config0.other_child_field
|
52
|
+
assert_nil created_child_config1.other_child_field
|
53
|
+
end
|
54
|
+
|
55
|
+
end
|
56
|
+
end
|
@@ -0,0 +1,65 @@
|
|
1
|
+
require_relative "test_helper"
|
2
|
+
module ActiveWorker
|
3
|
+
module JobQueue
|
4
|
+
class JobExecuterTest < ActiveSupport::TestCase
|
5
|
+
|
6
|
+
class TestClass
|
7
|
+
def self.test_method(param1, param2)
|
8
|
+
|
9
|
+
end
|
10
|
+
|
11
|
+
def self.raise_method
|
12
|
+
raise SignalException.new "SIGHUP"
|
13
|
+
end
|
14
|
+
|
15
|
+
def self.handle_error(e, method, params)
|
16
|
+
|
17
|
+
end
|
18
|
+
end
|
19
|
+
|
20
|
+
setup do
|
21
|
+
JobExecuter.stubs(:log_error)
|
22
|
+
end
|
23
|
+
|
24
|
+
test "can execute command from args" do
|
25
|
+
param1 = 1
|
26
|
+
param2 = 2
|
27
|
+
TestClass.expects(:test_method).with(param1, param2)
|
28
|
+
|
29
|
+
class_name = TestClass.to_s
|
30
|
+
method = :test_method
|
31
|
+
params = [param1,param2]
|
32
|
+
|
33
|
+
args = {}
|
34
|
+
args["class_name"] = class_name
|
35
|
+
args["method"] = method
|
36
|
+
args["params"] = params
|
37
|
+
|
38
|
+
JobExecuter.execute_task_from_args(args)
|
39
|
+
end
|
40
|
+
|
41
|
+
test "handles and reraises signal exception" do
|
42
|
+
class_name = TestClass.to_s
|
43
|
+
method = :raise_method
|
44
|
+
|
45
|
+
args = {}
|
46
|
+
args["class_name"] = class_name
|
47
|
+
args["method"] = method
|
48
|
+
|
49
|
+
assert_raise SignalException do
|
50
|
+
JobExecuter.execute_task_from_args(args)
|
51
|
+
end
|
52
|
+
end
|
53
|
+
|
54
|
+
test "can handle termination" do
|
55
|
+
params = [1,2]
|
56
|
+
fake_class = mock
|
57
|
+
fake_class.expects(:handle_termination).with(params)
|
58
|
+
|
59
|
+
JobExecuter.expects(:exit)
|
60
|
+
JobExecuter.handle_termination(fake_class, params)
|
61
|
+
end
|
62
|
+
|
63
|
+
end
|
64
|
+
end
|
65
|
+
end
|
@@ -0,0 +1,106 @@
|
|
1
|
+
require_relative "test_helper"
|
2
|
+
module ActiveWorker
|
3
|
+
module JobQueue
|
4
|
+
class QueueManagerTest < ActiveSupport::TestCase
|
5
|
+
|
6
|
+
test "can extract configuration id from worker" do
|
7
|
+
config_id = 5
|
8
|
+
worker = mock_worker(config_id)
|
9
|
+
|
10
|
+
extracted_id = QueueManager.new.configuration_id_from_worker(worker)
|
11
|
+
assert_equal config_id, extracted_id
|
12
|
+
end
|
13
|
+
|
14
|
+
test "can create job hash from worker" do
|
15
|
+
config_id = 5
|
16
|
+
worker = mock_worker(config_id)
|
17
|
+
|
18
|
+
job_hash = QueueManager.new.create_job_hash_from_worker(worker)
|
19
|
+
|
20
|
+
expected_hash = {"host" => "localhost",
|
21
|
+
"queues" => ["execute", "localhost_execute"],
|
22
|
+
"pid" => config_id.to_s,
|
23
|
+
"args" => {"params" => [config_id]}}
|
24
|
+
|
25
|
+
assert_equal expected_hash, job_hash
|
26
|
+
end
|
27
|
+
|
28
|
+
test "can get list of active jobs for configurations" do
|
29
|
+
manager = QueueManager.new
|
30
|
+
|
31
|
+
Resque.expects(:working).returns(mock_workers)
|
32
|
+
|
33
|
+
jobs = manager.active_jobs_for_configurations(configuration_ids)
|
34
|
+
|
35
|
+
assert_equal 4, jobs.size
|
36
|
+
assert_equal [1, 2, 3, 4], jobs.map { |j| j["pid"].to_i }
|
37
|
+
|
38
|
+
end
|
39
|
+
|
40
|
+
test "does not report on jobless workers" do
|
41
|
+
config_id = 5
|
42
|
+
worker = mock_worker(config_id, {})
|
43
|
+
|
44
|
+
job_hash = QueueManager.new.create_job_hash_from_worker(worker)
|
45
|
+
|
46
|
+
assert_nil job_hash
|
47
|
+
|
48
|
+
end
|
49
|
+
|
50
|
+
test "active workers can lose their jobs" do
|
51
|
+
manager = QueueManager.new
|
52
|
+
|
53
|
+
workers = mock_workers + [mock_worker(11, {})]
|
54
|
+
|
55
|
+
Resque.expects(:working).returns(workers)
|
56
|
+
|
57
|
+
ids = configuration_ids + [11]
|
58
|
+
|
59
|
+
jobs = manager.active_jobs_for_configurations(ids)
|
60
|
+
|
61
|
+
assert_equal 4, jobs.size
|
62
|
+
assert_equal [1, 2, 3, 4], jobs.map { |j| j["pid"].to_i }
|
63
|
+
end
|
64
|
+
|
65
|
+
test "does not return workers without jobs" do
|
66
|
+
manager = QueueManager.new
|
67
|
+
|
68
|
+
workers = mock_workers(4) + [mock_worker(11, {})]
|
69
|
+
|
70
|
+
Resque.expects(:working).returns(workers)
|
71
|
+
|
72
|
+
jobs = manager.active_jobs
|
73
|
+
|
74
|
+
assert_equal 4, jobs.size
|
75
|
+
assert_equal [0, 1, 2, 3], jobs.map { |j| j["pid"].to_i }
|
76
|
+
end
|
77
|
+
|
78
|
+
def mock_workers(num_workers = 10)
|
79
|
+
workers = []
|
80
|
+
num_workers.times do |num|
|
81
|
+
workers << mock_worker(num)
|
82
|
+
end
|
83
|
+
workers
|
84
|
+
end
|
85
|
+
|
86
|
+
def configuration_ids
|
87
|
+
[1, 2, 3, 4, 20]
|
88
|
+
end
|
89
|
+
|
90
|
+
def mock_worker(config_id, job = mock_job(config_id))
|
91
|
+
worker = mock
|
92
|
+
worker.stubs(:job).returns(job)
|
93
|
+
worker.stubs(:hostname).returns("bad_hostname")
|
94
|
+
worker.stubs(:to_s).returns("localhost:#{config_id}:execute,localhost_execute")
|
95
|
+
worker
|
96
|
+
end
|
97
|
+
|
98
|
+
def mock_job(config_id)
|
99
|
+
args = [{"params" => [config_id]}]
|
100
|
+
job = {"payload" => {"args" => args}}
|
101
|
+
end
|
102
|
+
|
103
|
+
|
104
|
+
end
|
105
|
+
end
|
106
|
+
end
|