tochtli 0.5.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (62) hide show
  1. checksums.yaml +7 -0
  2. data/.travis.yml +14 -0
  3. data/Gemfile +32 -0
  4. data/History.md +138 -0
  5. data/README.md +46 -0
  6. data/Rakefile +50 -0
  7. data/VERSION +1 -0
  8. data/assets/communication.png +0 -0
  9. data/assets/layers.png +0 -0
  10. data/examples/01-screencap-service/Gemfile +3 -0
  11. data/examples/01-screencap-service/README.md +5 -0
  12. data/examples/01-screencap-service/client.rb +15 -0
  13. data/examples/01-screencap-service/common.rb +15 -0
  14. data/examples/01-screencap-service/server.rb +26 -0
  15. data/examples/02-log-analyzer/Gemfile +3 -0
  16. data/examples/02-log-analyzer/README.md +5 -0
  17. data/examples/02-log-analyzer/client.rb +95 -0
  18. data/examples/02-log-analyzer/common.rb +33 -0
  19. data/examples/02-log-analyzer/sample.log +10001 -0
  20. data/examples/02-log-analyzer/server.rb +133 -0
  21. data/lib/tochtli.rb +177 -0
  22. data/lib/tochtli/active_record_connection_cleaner.rb +9 -0
  23. data/lib/tochtli/application.rb +135 -0
  24. data/lib/tochtli/base_client.rb +135 -0
  25. data/lib/tochtli/base_controller.rb +360 -0
  26. data/lib/tochtli/controller_manager.rb +99 -0
  27. data/lib/tochtli/engine.rb +15 -0
  28. data/lib/tochtli/message.rb +114 -0
  29. data/lib/tochtli/rabbit_client.rb +36 -0
  30. data/lib/tochtli/rabbit_connection.rb +249 -0
  31. data/lib/tochtli/reply_queue.rb +129 -0
  32. data/lib/tochtli/simple_validation.rb +23 -0
  33. data/lib/tochtli/test.rb +9 -0
  34. data/lib/tochtli/test/client.rb +28 -0
  35. data/lib/tochtli/test/controller.rb +66 -0
  36. data/lib/tochtli/test/integration.rb +78 -0
  37. data/lib/tochtli/test/memory_cache.rb +22 -0
  38. data/lib/tochtli/test/test_case.rb +191 -0
  39. data/lib/tochtli/test/test_unit.rb +22 -0
  40. data/lib/tochtli/version.rb +3 -0
  41. data/log_generator.rb +11 -0
  42. data/test/base_client_test.rb +68 -0
  43. data/test/controller_functional_test.rb +87 -0
  44. data/test/controller_integration_test.rb +274 -0
  45. data/test/controller_manager_test.rb +75 -0
  46. data/test/dummy/Rakefile +7 -0
  47. data/test/dummy/config/application.rb +36 -0
  48. data/test/dummy/config/boot.rb +4 -0
  49. data/test/dummy/config/database.yml +3 -0
  50. data/test/dummy/config/environment.rb +5 -0
  51. data/test/dummy/config/rabbit.yml +4 -0
  52. data/test/dummy/db/.gitkeep +0 -0
  53. data/test/dummy/log/.gitkeep +0 -0
  54. data/test/key_matcher_test.rb +100 -0
  55. data/test/log/.gitkeep +0 -0
  56. data/test/message_test.rb +80 -0
  57. data/test/rabbit_client_test.rb +71 -0
  58. data/test/rabbit_connection_test.rb +151 -0
  59. data/test/test_helper.rb +32 -0
  60. data/test/version_test.rb +8 -0
  61. data/tochtli.gemspec +129 -0
  62. metadata +259 -0
@@ -0,0 +1,75 @@
1
+ require_relative 'test_helper'
2
+
3
+ Thread.abort_on_exception = true
4
+
5
+ class ControllerManagerTest < Minitest::Test
6
+ include Tochtli::Test::Helpers
7
+
8
+ class FirstController < Tochtli::BaseController
9
+ end
10
+
11
+ class SecondController < Tochtli::BaseController
12
+ end
13
+
14
+ class ThirdController < Tochtli::BaseController
15
+ end
16
+
17
+ def setup
18
+ @logger = Logger.new(STDERR)
19
+ @logger.level = Logger::WARN
20
+ end
21
+
22
+ def teardown
23
+ Tochtli::ControllerManager.stop
24
+ end
25
+
26
+ def test_start_single_controller
27
+ Tochtli::ControllerManager.start(FirstController, connection: @connection, logger: @logger)
28
+ assert FirstController.started?
29
+ refute SecondController.started?
30
+ refute ThirdController.started?
31
+ end
32
+
33
+ def test_start_selected_controllers
34
+ Tochtli::ControllerManager.start(FirstController, ThirdController, connection: @connection, logger: @logger)
35
+ assert FirstController.started?
36
+ refute SecondController.started?
37
+ assert ThirdController.started?
38
+ end
39
+
40
+ def test_start_all_controllers
41
+ Tochtli::ControllerManager.start(:all, connection: @connection, logger: @logger)
42
+ assert FirstController.started?
43
+ assert SecondController.started?
44
+ assert ThirdController.started?
45
+ end
46
+
47
+ def test_restart_only_active_controllers
48
+ Tochtli::ControllerManager.start(FirstController, SecondController, connection: @connection, logger: @logger)
49
+ Tochtli::ControllerManager.restart(connection: @connection, logger: @logger)
50
+ assert FirstController.started?
51
+ assert SecondController.started?
52
+ refute ThirdController.started?
53
+ end
54
+
55
+ def test_state_monitor
56
+ Tochtli::ControllerManager.start(FirstController, connection: @connection, logger: @logger)
57
+ end
58
+
59
+ def test_multiple_queues
60
+ Tochtli::ControllerManager.setup(connection: @connection, logger: @logger)
61
+ Tochtli::ControllerManager.start(FirstController, queue_name: 'first_queue')
62
+ Tochtli::ControllerManager.start(FirstController, queue_name: 'second_queue')
63
+
64
+ assert_equal ["first_queue", "second_queue"], FirstController.dispatcher.queues.map(&:name).sort
65
+ end
66
+
67
+ def test_restart_multiple_queues
68
+ Tochtli::ControllerManager.setup(connection: @connection, logger: @logger)
69
+ Tochtli::ControllerManager.start(FirstController, queue_name: 'first_queue')
70
+ Tochtli::ControllerManager.start(FirstController, queue_name: 'second_queue')
71
+ Tochtli::ControllerManager.restart
72
+
73
+ assert_equal ["first_queue", "second_queue"], FirstController.dispatcher.queues.map(&:name).sort
74
+ end
75
+ end
@@ -0,0 +1,7 @@
1
+ #!/usr/bin/env rake
2
+ # Add your own tasks in files placed in lib/tasks ending in .rake,
3
+ # for example lib/tasks/capistrano.rake, and they will automatically be available to Rake.
4
+
5
+ require File.expand_path('../config/application', __FILE__)
6
+
7
+ Dummy::Application.load_tasks
@@ -0,0 +1,36 @@
1
+ require File.expand_path('../boot', __FILE__)
2
+
3
+ require 'rails/all'
4
+
5
+ Bundler.require
6
+
7
+ require 'tochtli'
8
+
9
+ module Dummy
10
+ class Application < Rails::Application
11
+ # Set dummy app root
12
+ config.root = File.expand_path('../..', __FILE__)
13
+
14
+ config.eager_load = false
15
+ config.encoding = "utf-8"
16
+ config.filter_parameters += [:password]
17
+ config.active_support.escape_html_entities_in_json = true
18
+ config.active_record.schema_format = :sql
19
+ config.assets.enabled = true
20
+ config.assets.version = '1.0'
21
+ config.cache_classes = true
22
+ config.serve_static_files = true
23
+ config.static_cache_control = "public, max-age=3600"
24
+ config.whiny_nils = true
25
+ config.consider_all_requests_local = true
26
+ config.action_controller.perform_caching = false
27
+ config.action_dispatch.show_exceptions = false
28
+ config.action_controller.allow_forgery_protection = false
29
+ config.action_mailer.delivery_method = :test
30
+ config.active_support.deprecation = :stderr
31
+ config.secret_token = 'efc39d860d9d26146e6546fc69b12c014f98785e08a8099174583af0a04a27774604060f591678aef27c491f9d00792a00884a92bb35e4ca122d0cbeddd4ea98'
32
+ config.active_support.test_order = :random
33
+ end
34
+ end
35
+
36
+
@@ -0,0 +1,4 @@
1
+ require 'rubygems'
2
+
3
+ ENV['BUNDLE_GEMFILE'] = File.expand_path('../../../../Gemfile', __FILE__)
4
+ require 'bundler/setup'
@@ -0,0 +1,3 @@
1
+ test:
2
+ adapter: sqlite3
3
+ database: db/test.sqlite3
@@ -0,0 +1,5 @@
1
+ # Load the rails application
2
+ require File.expand_path('../application', __FILE__)
3
+
4
+ # Initialize the rails application
5
+ Dummy::Application.initialize!
@@ -0,0 +1,4 @@
1
+ test:
2
+ host: 127.0.0.1
3
+ user: guest
4
+ password: guest
File without changes
File without changes
@@ -0,0 +1,100 @@
1
+ require_relative 'test_helper'
2
+ require 'benchmark'
3
+
4
+ class MessageTest < Minitest::Test
5
+ KeyPattern = Tochtli::BaseController::KeyPattern
6
+
7
+ def test_simple_pattern
8
+ pattern = KeyPattern.new('a.b.c')
9
+
10
+ assert_matches pattern, 'a.b.c'
11
+ refute_matches pattern, 'a.b.c.d'
12
+ refute_matches pattern, 'b.c.d'
13
+ end
14
+
15
+ def test_asterix_at_start
16
+ pattern = KeyPattern.new('*.b.c')
17
+
18
+ assert_matches pattern, 'a.b.c'
19
+ assert_matches pattern, 'b.b.c'
20
+ refute_matches pattern, 'a.b.c.d'
21
+ end
22
+
23
+ def test_asterix_in_the_middle
24
+ pattern = KeyPattern.new('a.*.b.c')
25
+
26
+ assert_matches pattern, 'a.a.b.c'
27
+ assert_matches pattern, 'a.d.b.c'
28
+ refute_matches pattern, 'a.b.c.d'
29
+ end
30
+
31
+ def test_asterix_at_the_end
32
+ pattern = KeyPattern.new('a.b.c.*')
33
+
34
+ assert_matches pattern, 'a.b.c.d'
35
+ assert_matches pattern, 'a.b.c.a'
36
+ refute_matches pattern, 'a.b.c'
37
+ refute_matches pattern, 'a.b.c.d.e'
38
+ end
39
+
40
+ def test_hash
41
+ pattern = KeyPattern.new('#')
42
+
43
+ assert_matches pattern, ''
44
+ assert_matches pattern, 'a.b.c'
45
+ assert_matches pattern, 'a.b.b.c'
46
+ end
47
+
48
+ def test_hash_at_start
49
+ pattern = KeyPattern.new('#.b.c')
50
+
51
+ assert_matches pattern, 'b.c'
52
+ assert_matches pattern, 'a.b.c'
53
+ assert_matches pattern, 'a.b.b.c'
54
+ refute_matches pattern, 'a.b.c.d'
55
+ end
56
+
57
+ def test_hash_in_the_middle
58
+ pattern = KeyPattern.new('a.#.c')
59
+
60
+ assert_matches pattern, 'a.a.b.c'
61
+ assert_matches pattern, 'a.d.b.c'
62
+ refute_matches pattern, 'a.b.c.d'
63
+ end
64
+
65
+ def test_hash_at_the_end
66
+ pattern = KeyPattern.new('a.b.#')
67
+
68
+ assert_matches pattern, 'a.b.c.d'
69
+ assert_matches pattern, 'a.b.c'
70
+ assert_matches pattern, 'a.b.c.d.e'
71
+ end
72
+
73
+ def test_complex
74
+ pattern = KeyPattern.new('*.*.a.b.#.c.#')
75
+
76
+ assert_matches pattern, '1.2.a.b.c.d'
77
+ assert_matches pattern, '1.2.a.b.3.4.c.d'
78
+ assert_matches pattern, '1.2.a.b.c'
79
+ refute_matches pattern, 'a.b.c.d.e'
80
+ end
81
+
82
+ def test_performance
83
+ pattern = KeyPattern.new('*.*.a.b.#.c.#')
84
+
85
+ n = 10_000
86
+ time = Benchmark.realtime { n.times { pattern =~ '1.2.a.b.3.4.c.d' } }
87
+
88
+ assert time < 0.1
89
+ end
90
+
91
+ protected
92
+
93
+ def assert_matches(pattern, key)
94
+ assert pattern =~ key, "#{key} SHOULD match #{pattern}"
95
+ end
96
+
97
+ def refute_matches(pattern, key)
98
+ assert pattern !~ key, "#{key} MUST NOT match #{pattern}"
99
+ end
100
+ end
File without changes
@@ -0,0 +1,80 @@
1
+ require_relative 'test_helper'
2
+
3
+ class MessageTest < Minitest::Test
4
+ include Tochtli::Test::Helpers
5
+
6
+ class SimpleMessage < Tochtli::Message
7
+ route_to 'test.controller.simple'
8
+
9
+ attribute :text, String
10
+ attribute :timestamp, Time
11
+ attribute :optional, String, required: false
12
+
13
+ def validate
14
+ setup_timestamp
15
+ unless optional.nil? || optional =~ /\A[a-z!]+\z/i
16
+ add_error "Invalid optional attribute: #{optional}"
17
+ end
18
+ super
19
+ end
20
+
21
+ def setup_timestamp
22
+ @timestamp ||= Time.now
23
+ end
24
+ end
25
+
26
+ class OpenMessage < Tochtli::Message
27
+ ignore_extra_attributes
28
+
29
+ attribute :text, String
30
+ end
31
+
32
+ def test_routing_key
33
+ assert_equal 'test.controller.simple', SimpleMessage.routing_key
34
+ end
35
+
36
+ def test_simple_message_without_optional
37
+ message = SimpleMessage.new(text: 'Hello')
38
+ assert_equal 'Hello', message.text
39
+ assert_nil message.optional
40
+ assert message.valid?
41
+ end
42
+
43
+ def test_simple_message_with_optional
44
+ message = SimpleMessage.new(text: 'Hello', optional: 'world!')
45
+ assert_equal 'Hello', message.text
46
+ assert_equal 'world!', message.optional
47
+ assert message.valid?
48
+ end
49
+
50
+ def test_invalid_attribute
51
+ message = SimpleMessage.new(text: 'Hello', optional: 'world 123')
52
+ assert message.invalid?, "Message passed validation when it should not"
53
+ assert_equal "Invalid optional attribute: world 123", message.errors[0]
54
+ end
55
+
56
+ def test_validation_callback_without_value
57
+ message = SimpleMessage.new(text: 'Hello')
58
+ assert message.valid?
59
+ assert_kind_of Time, message.timestamp
60
+ end
61
+
62
+ def test_validation_callbacks_with_given_value
63
+ timestamp = Time.at(0) # long long ago
64
+ message = SimpleMessage.new(text: 'Hello', timestamp: timestamp)
65
+ assert message.valid?
66
+ assert_equal timestamp, message.timestamp
67
+ end
68
+
69
+ def test_undefined_attribute_error
70
+ message = SimpleMessage.new(text: 'Hello', extra: 'from Paris')
71
+ assert message.invalid? # Undefined attribute :extra
72
+ assert_equal "Unexpected attributes: extra", message.errors[0]
73
+ end
74
+
75
+ def test_ignore_excess_attribute
76
+ message = OpenMessage.new(text: 'Hello', extra: 'from Paris')
77
+ assert message.valid?
78
+ end
79
+
80
+ end
@@ -0,0 +1,71 @@
1
+ require_relative 'test_helper'
2
+
3
+ class RabbitClientTest < Minitest::Test
4
+ include Tochtli::Test::Client
5
+
6
+ def test_reply_queue
7
+ reply_queue = @client.reply_queue
8
+ assert_kind_of Tochtli::ReplyQueue, reply_queue
9
+ assert_equal @client.rabbit_connection, reply_queue.connection
10
+ refute_nil reply_queue.name
11
+ end
12
+
13
+ def test_publishing
14
+ @client.publish FakeMessage.new(test_attr: 'test')
15
+
16
+ assert_published FakeMessage, test_attr: 'test'
17
+ end
18
+
19
+ def test_reply
20
+ handler = Tochtli::Test::TestMessageHandler.new
21
+
22
+ message = FakeMessage.new(test_attr: 'test')
23
+ @client.publish message, handler: handler
24
+
25
+ expected_reply = handle_reply(FakeReply, message, result: 'test123')
26
+
27
+ assert_equal expected_reply, handler.reply
28
+ end
29
+
30
+ def test_reply_timeout
31
+ handler = Tochtli::Test::TestMessageHandler.new
32
+ message = FakeMessage.new(test_attr: 'test')
33
+ @client.publish message, handler: handler, timeout: 0.05
34
+ sleep 0.1
35
+ assert_equal message, handler.timeout_message
36
+ end
37
+
38
+ def test_reply_no_timeout
39
+ handler = Tochtli::Test::TestMessageHandler.new
40
+ message = FakeMessage.new(test_attr: 'test')
41
+ @client.publish message, handler: handler, timeout: 0.1
42
+
43
+ expected_reply = create_reply(FakeReply, message, result: 'test123')
44
+ @client.reply_queue.handle_reply expected_reply
45
+
46
+ sleep 0.2
47
+
48
+ assert_equal expected_reply, handler.reply
49
+ assert_nil handler.timeout_message
50
+ end
51
+
52
+ def test_message_drop
53
+ handler = Tochtli::Test::TestMessageHandler.new
54
+ message = FakeMessage.new(test_attr: 'test')
55
+ @client.publish message, handler: handler, timeout: 0.1
56
+ @client.reply_queue.handle_reply Tochtli::MessageDropped.new("Message dropped", message), message.id
57
+
58
+ assert_kind_of Tochtli::MessageDropped, handler.error
59
+ end
60
+
61
+ class FakeMessage < Tochtli::Message
62
+ route_to 'test.fake.topic'
63
+
64
+ attributes :test_attr
65
+ end
66
+
67
+ class FakeReply < Tochtli::Message
68
+ attributes :result
69
+ end
70
+
71
+ end
@@ -0,0 +1,151 @@
1
+ require_relative 'test_helper'
2
+ require 'tochtli/test/test_case'
3
+
4
+ class RabbitConnectionTest < Minitest::Test
5
+ def setup
6
+ Tochtli::RabbitConnection.close('test')
7
+ end
8
+
9
+ def teardown
10
+ Tochtli::RabbitConnection.close('test')
11
+ end
12
+
13
+ class TestMessage < Tochtli::Message
14
+ attributes :text
15
+ end
16
+
17
+ def test_connection_with_default_options
18
+ Tochtli::RabbitConnection.open('test') do |connection|
19
+ assert_equal "puzzleflow.services", connection.exchange.name
20
+ end
21
+ end
22
+
23
+ def test_connection_with_custom_options
24
+ Tochtli::RabbitConnection.open('test', exchange_name: "puzzleflow.tests") do |connection|
25
+ assert_equal "puzzleflow.tests", connection.exchange.name
26
+ end
27
+ end
28
+
29
+ def test_multiple_channels_and_exchanges
30
+ Tochtli::RabbitConnection.open('test', exchange_name: "puzzleflow.tests") do |connection|
31
+ another_thread = Thread.new {}
32
+
33
+ current_channel = connection.channel
34
+ another_channel = connection.channel(another_thread)
35
+
36
+ current_exchange = connection.exchange
37
+ another_exchange = connection.exchange(another_thread)
38
+
39
+ refute_equal current_channel, another_channel
40
+ refute_equal current_exchange, another_exchange
41
+ assert_equal "puzzleflow.tests", current_exchange.name
42
+ assert_equal "puzzleflow.tests", another_exchange.name
43
+ end
44
+ end
45
+
46
+ def test_queue_creation_and_existance
47
+ Tochtli::RabbitConnection.open('test') do |connection|
48
+ queue = connection.queue('test-queue', [], auto_delete: true)
49
+ refute_nil queue
50
+ assert_equal 'test-queue', queue.name
51
+ assert connection.queue_exists?('test-queue')
52
+ end
53
+ end
54
+
55
+ def test_reply_queue_recovery
56
+ Tochtli::RabbitConnection.open('test',
57
+ network_recovery_interval: 0.1,
58
+ recover_from_connection_close: true) do |rabbit_connection|
59
+ reply_queue = rabbit_connection.reply_queue
60
+ original_name = reply_queue.name
61
+ timeout = 0.3
62
+
63
+ message = TestMessage.new(text: "Response")
64
+ reply = TestMessage.new(text: "Reply")
65
+ handler = Tochtli::Test::TestMessageHandler.new
66
+ reply_queue.register_message_handler message, handler, timeout
67
+
68
+ rabbit_connection.publish reply_queue.name, reply, correlation_id: message.id, timeout: timeout
69
+ sleep timeout
70
+
71
+ refute_nil handler.reply
72
+
73
+ # simulate network failure
74
+ rabbit_connection.connection.handle_network_failure(RuntimeError.new('fake connection error'))
75
+ sleep 0.1 until rabbit_connection.open? # wait for recovery
76
+ refute_equal original_name, reply_queue.name, "Recovered queue should have re-generated name"
77
+
78
+ message = TestMessage.new(text: "Response")
79
+ reply = TestMessage.new(text: "Reply")
80
+ handler = Tochtli::Test::TestMessageHandler.new
81
+ reply_queue.register_message_handler message, handler, timeout
82
+
83
+ rabbit_connection.publish reply_queue.name, reply, correlation_id: message.id, timeout: timeout
84
+ sleep timeout
85
+
86
+ refute_nil handler.reply
87
+ end
88
+ end
89
+
90
+ def test_multithreaded_consumer_performance
91
+ work_pool_size = 10
92
+ Tochtli::RabbitConnection.open('test',
93
+ exchange_name: "puzzleflow.tests",
94
+ work_pool_size: work_pool_size) do |connection|
95
+ mutex = Mutex.new
96
+ cv = ConditionVariable.new
97
+ thread_count = 5
98
+ message_count = 100
99
+ expected_message_count = message_count*thread_count
100
+
101
+ consumed = 0
102
+ consumed_mutex = Mutex.new
103
+ consumer_threads = Set.new
104
+ consumer = Proc.new do |delivery_info, metadata, payload|
105
+ consumed_mutex.synchronize { consumed += 1 }
106
+ consumer_threads << Thread.current
107
+ connection.publish metadata.reply_to, TestMessage.new(text: "Response to #{payload}")
108
+ end
109
+
110
+ queue = connection.channel.queue('', auto_delete: true)
111
+ queue.bind(connection.exchange, routing_key: queue.name)
112
+ queue.subscribe(block: false, &consumer)
113
+
114
+ replies = 0
115
+ reply_consumer = Proc.new do |delivery_info, metadata, payload|
116
+ replies += 1
117
+ mutex.synchronize { cv.signal } if replies == expected_message_count
118
+ end
119
+
120
+ reply_queue = connection.channel.queue('', auto_delete: true)
121
+ reply_queue.bind(connection.exchange, routing_key: reply_queue.name)
122
+ reply_queue.subscribe(block: false, &reply_consumer)
123
+
124
+ start_t = Time.now
125
+
126
+ threads = (1..thread_count).collect do
127
+ t = Thread.new do
128
+ message_count.times do |i|
129
+ connection.publish queue.name, TestMessage.new(text: "Message #{i}"),
130
+ reply_to: reply_queue.name
131
+ end
132
+ end
133
+ t.abort_on_exception = true
134
+ t
135
+ end
136
+
137
+ threads.each(&:join)
138
+
139
+ mutex.synchronize { cv.wait(mutex, 5.0) }
140
+
141
+ end_t = Time.now
142
+ time = end_t - start_t
143
+
144
+ assert_equal expected_message_count, consumed
145
+ assert_equal expected_message_count, replies
146
+ assert_equal work_pool_size, consumer_threads.size
147
+
148
+ puts "Published: #{expected_message_count} in #{time} (#{expected_message_count/time}req/s)"
149
+ end
150
+ end
151
+ end