skein 0.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (47) hide show
  1. checksums.yaml +7 -0
  2. data/Gemfile +9 -0
  3. data/Gemfile.lock +69 -0
  4. data/README.md +57 -0
  5. data/RELEASES.md +4 -0
  6. data/Rakefile +30 -0
  7. data/VERSION +1 -0
  8. data/bin/skein +186 -0
  9. data/config/.gitignore +3 -0
  10. data/config/skein.yml.example +11 -0
  11. data/lib/skein.rb +24 -0
  12. data/lib/skein/client.rb +51 -0
  13. data/lib/skein/client/publisher.rb +14 -0
  14. data/lib/skein/client/rpc.rb +96 -0
  15. data/lib/skein/client/subscriber.rb +25 -0
  16. data/lib/skein/client/worker.rb +51 -0
  17. data/lib/skein/config.rb +87 -0
  18. data/lib/skein/connected.rb +52 -0
  19. data/lib/skein/context.rb +38 -0
  20. data/lib/skein/handler.rb +86 -0
  21. data/lib/skein/handler/async.rb +9 -0
  22. data/lib/skein/handler/threaded.rb +7 -0
  23. data/lib/skein/rabbitmq.rb +44 -0
  24. data/lib/skein/reporter.rb +11 -0
  25. data/lib/skein/rpc.rb +24 -0
  26. data/lib/skein/rpc/base.rb +23 -0
  27. data/lib/skein/rpc/error.rb +34 -0
  28. data/lib/skein/rpc/notification.rb +2 -0
  29. data/lib/skein/rpc/request.rb +62 -0
  30. data/lib/skein/rpc/response.rb +38 -0
  31. data/lib/skein/support.rb +67 -0
  32. data/skein.gemspec +95 -0
  33. data/test/data/sample_config.yml +13 -0
  34. data/test/helper.rb +42 -0
  35. data/test/script/em_example +28 -0
  36. data/test/unit/test_skein_client.rb +18 -0
  37. data/test/unit/test_skein_client_publisher.rb +10 -0
  38. data/test/unit/test_skein_client_subscriber.rb +41 -0
  39. data/test/unit/test_skein_client_worker.rb +61 -0
  40. data/test/unit/test_skein_config.rb +33 -0
  41. data/test/unit/test_skein_context.rb +44 -0
  42. data/test/unit/test_skein_rabbitmq.rb +14 -0
  43. data/test/unit/test_skein_reporter.rb +4 -0
  44. data/test/unit/test_skein_rpc_error.rb +10 -0
  45. data/test/unit/test_skein_rpc_request.rb +93 -0
  46. data/test/unit/test_skein_support.rb +95 -0
  47. metadata +148 -0
@@ -0,0 +1,14 @@
1
+ class Skein::Client::Publisher < Skein::Connected
2
+ # == Instance Methods =====================================================
3
+
4
+ def initialize(queue_name, connection: nil, context: nil)
5
+ super(connection: connection, context: context)
6
+
7
+ @queue = self.channel.topic(queue_name)
8
+ end
9
+
10
+ def publish!(message, routing_key = nil)
11
+ @queue.publish(JSON.dump(message), routing_key: routing_key)
12
+ end
13
+ alias_method :<<, :publish!
14
+ end
@@ -0,0 +1,96 @@
1
+ require 'securerandom'
2
+ require 'fiber'
3
+
4
+ class Skein::Client::RPC < Skein::Connected
5
+ # == Constants ============================================================
6
+
7
+ QUEUE_NAME_DEFAULT = 'skein_rpc'.freeze
8
+
9
+ # == Properties ===========================================================
10
+
11
+ # == Instance Methods =====================================================
12
+
13
+ def initialize(queue_name = nil, connection: nil, context: nil)
14
+ super(connection: connection, context: context)
15
+
16
+ @rpc_queue = self.channel.queue(queue_name || QUEUE_NAME_DEFAULT, durable: true)
17
+ @response_queue = self.channel.queue(@ident, durable: true, header: true, auto_delete: true)
18
+
19
+ @callback = { }
20
+
21
+ @consumer = @response_queue.subscribe do |metadata, payload, extra|
22
+ # FIX: Deal with mixup between Bunny and MarchHare
23
+ # puts [metadata,payload,extra].inspect
24
+
25
+ # puts [metadata,payload,extra].map(&:class).inspect
26
+
27
+ if (extra)
28
+ payload = extra
29
+ elsif (!payload)
30
+ payload = metadata
31
+ end
32
+
33
+ begin
34
+ response = JSON.load(payload)
35
+
36
+ if (callback = @callback.delete(response['id']))
37
+ case (callback)
38
+ when Queue
39
+ callback << response['result']
40
+ when Proc
41
+ callback.call
42
+ end
43
+ end
44
+ rescue => e
45
+ # FIX: Error handling
46
+ puts e.inspect
47
+ end
48
+ end
49
+ end
50
+
51
+ def close
52
+ @consumer and @consumer.cancel
53
+ @consumer = nil
54
+
55
+ super
56
+ end
57
+
58
+ def method_missing(name, *args)
59
+ name = name.to_s
60
+
61
+ blocking = !name.sub!(/!\z/, '')
62
+
63
+ message_id = SecureRandom.uuid
64
+ request = JSON.dump(
65
+ method: name,
66
+ params: args,
67
+ id: message_id
68
+ )
69
+
70
+ @channel.default_exchange.publish(
71
+ request,
72
+ routing_key: @rpc_queue.name,
73
+ reply_to: blocking ? @ident : nil,
74
+ message_id: message_id
75
+ )
76
+
77
+ if (block_given?)
78
+ @callback[message_id] =
79
+ if (defined?(EventMachine))
80
+ EventMachine.next_tick do
81
+ yield
82
+ end
83
+ else
84
+ lambda do
85
+ yield
86
+ end
87
+ end
88
+ elsif (blocking)
89
+ queue = Queue.new
90
+
91
+ @callback[message_id] = queue
92
+
93
+ queue.pop
94
+ end
95
+ end
96
+ end
@@ -0,0 +1,25 @@
1
+ class Skein::Client::Subscriber < Skein::Connected
2
+ # == Instance Methods =====================================================
3
+
4
+ def initialize(queue_name, routing_key = nil, connection: nil, context: nil)
5
+ super(connection: connection, context: context)
6
+
7
+ @queue = self.channel.topic(queue_name)
8
+ @subscribe_queue = self.channel.queue('', exclusive: true)
9
+
10
+ @subscribe_queue.bind(@queue, routing_key: routing_key)
11
+ end
12
+
13
+ def listen
14
+ case (@subscribe_queue.class.to_s.split(/::/)[0])
15
+ when 'Bunny'
16
+ @subscribe_queue.subscribe(block: true) do |delivery_info, properties, payload|
17
+ yield(JSON.load(payload), delivery_info, properties)
18
+ end
19
+ when 'MarchHare'
20
+ @subscribe_queue.subscribe(block: true) do |metadata, payload|
21
+ yield(JSON.load(payload), metadata)
22
+ end
23
+ end
24
+ end
25
+ end
@@ -0,0 +1,51 @@
1
+ require 'json'
2
+
3
+ class Skein::Client::Worker < Skein::Connected
4
+ # == Instance Methods =====================================================
5
+
6
+ def initialize(queue_name, connection: nil, context: nil)
7
+ super(connection: connection, context: context)
8
+
9
+ lock do
10
+ queue = self.channel.queue(queue_name, durable: true)
11
+
12
+ @handler = Skein::Handler.for(self)
13
+
14
+ @subscriber = Skein::Client::Subscriber.new(queue_name, connection: self.connection)
15
+
16
+ @thread = Thread.new do
17
+ @subscriber.listen do |payload, delivery_tag, reply_to|
18
+ @handler.handle(payload) do |reply_json|
19
+ channel.acknowledge(delivery_tag, true)
20
+
21
+ if (reply_to)
22
+ channel.default_exchange.publish(
23
+ reply_json,
24
+ routing_key: reply_to
25
+ )
26
+ end
27
+ end
28
+ end
29
+ end
30
+ end
31
+ end
32
+
33
+ def close
34
+ @thread.kill
35
+ @thread.join
36
+
37
+ super
38
+ end
39
+
40
+ def join
41
+ @thread and @thread.join
42
+ end
43
+
44
+ def async?
45
+ # Define this method as `true` in any subclass that requires async
46
+ # callback-style delegation.
47
+ false
48
+ end
49
+
50
+ protected
51
+ end
@@ -0,0 +1,87 @@
1
+ require 'ostruct'
2
+ require 'yaml'
3
+
4
+ class Skein::Config < OpenStruct
5
+ # == Constants ============================================================
6
+
7
+ CONFIG_PATH_DEFAULT = 'config/skein.yml'.freeze
8
+
9
+ ENV_DEFAULT = 'development'.freeze
10
+
11
+ DRIVERS = {
12
+ bunny: 'Bunny',
13
+ march_hare: 'MarchHare'
14
+ }.freeze
15
+
16
+ DRIVER_PLATFORM_DEFAULT = Hash.new(:bunny).merge(
17
+ "java" => :march_hare
18
+ ).freeze
19
+
20
+ DRIVER_DEFAULT = (
21
+ DRIVERS.find do |name, const|
22
+ const_defined?(const)
23
+ end || [ ]
24
+ )[0] || DRIVER_PLATFORM_DEFAULT[RUBY_PLATFORM]
25
+
26
+ DEFAULTS = {
27
+ host: '127.0.0.1',
28
+ port: 5672,
29
+ username: 'guest',
30
+ password: 'guest',
31
+ driver: DRIVER_DEFAULT,
32
+ namespace: nil
33
+ }.freeze
34
+
35
+ # == Class Methods ========================================================
36
+
37
+ def self.root
38
+ if (defined?(Rails))
39
+ Rails.root
40
+ else
41
+ Dir.pwd
42
+ end
43
+ end
44
+
45
+ def self.env
46
+ if (defined?(Rails))
47
+ Rails.env.to_s
48
+ else
49
+ ENV['RAILS_ENV'] || ENV_DEFAULT
50
+ end
51
+ end
52
+
53
+ def self.path
54
+ File.expand_path(CONFIG_PATH_DEFAULT, self.root)
55
+ end
56
+
57
+ def self.exist?
58
+ File.exist?(self.path)
59
+ end
60
+
61
+ # == Instance Methods =====================================================
62
+
63
+ def initialize(options = nil)
64
+ config_path = nil
65
+
66
+ case (options)
67
+ when String
68
+ if (File.exist?(options))
69
+ config_path = options
70
+ end
71
+ when Hash
72
+ super(options)
73
+
74
+ return
75
+ when false, :default
76
+ # Ignore configuration file, use defaults
77
+ else
78
+ config_path = File.expand_path('config/skein.yml', self.class.root)
79
+ end
80
+
81
+ if (config_path and File.exist?(config_path))
82
+ super(DEFAULTS.merge(YAML.load_file(config_path)[self.class.env] || { }))
83
+ else
84
+ super(DEFAULTS)
85
+ end
86
+ end
87
+ end
@@ -0,0 +1,52 @@
1
+ class Skein::Connected
2
+ # == Properties ===========================================================
3
+
4
+ attr_reader :context
5
+ attr_reader :ident
6
+ attr_reader :connection
7
+ attr_reader :channel
8
+
9
+ # == Instance Methods =====================================================
10
+
11
+ def initialize(connection: nil, context: nil)
12
+ @mutex = Mutex.new
13
+ @shared_connection = !!connection
14
+
15
+ @connection = connection || Skein::RabbitMQ.connect
16
+ @channel = @connection.create_channel
17
+
18
+ @context = context || Skein::Context.new
19
+ @ident = @context.ident(self)
20
+ end
21
+
22
+ def lock
23
+ @mutex.synchronize do
24
+ yield
25
+ end
26
+ end
27
+
28
+ def close
29
+ lock do
30
+ begin
31
+ @channel and @channel.close
32
+
33
+ rescue => e
34
+ if (defined?(MarchHare))
35
+ case e
36
+ when MarchHare::ChannelLevelException, MarchHare::ChannelAlreadyClosed
37
+ # Ignored since we're finished with the channel anyway
38
+ else
39
+ raise e
40
+ end
41
+ end
42
+ end
43
+
44
+ @channel = nil
45
+
46
+ unless (@shared_connection)
47
+ @connection and @connection.close
48
+ @connection = nil
49
+ end
50
+ end
51
+ end
52
+ end
@@ -0,0 +1,38 @@
1
+ class Skein::Context
2
+ # == Properties ===========================================================
3
+
4
+ attr_reader :hostname
5
+ attr_reader :process_name
6
+ attr_reader :process_id
7
+ attr_accessor :reporter
8
+
9
+ # == Class Methods ========================================================
10
+
11
+ def self.default
12
+ @default ||= self.new
13
+ end
14
+
15
+ # == Instance Methods =====================================================
16
+
17
+ def initialize(hostname: nil, process_name: nil, process_id: nil, config: nil)
18
+ @hostname = (hostname || Skein::Support.hostname).dup.freeze
19
+ @process_name = (process_name || Skein::Support.process_name).dup.freeze
20
+ @process_id = process_id || Skein::Support.process_id
21
+ end
22
+
23
+ def ident(object)
24
+ # FUTURE: Add pack/unpack methods for whatever format this ends up being
25
+ # so the components can be extracted by another application for
26
+ # diagnostic reasons.
27
+ '%s#%d+%s@%s' % [
28
+ @process_name,
29
+ @process_id,
30
+ object.object_id,
31
+ @hostname
32
+ ]
33
+ end
34
+
35
+ def exception!(*args)
36
+ @reporter and @reporter.exception!(*args)
37
+ end
38
+ end
@@ -0,0 +1,86 @@
1
+ class Skein::Handler
2
+ # == Class Methods ========================================================
3
+
4
+ def self.for(target)
5
+ case (target.respond_to?(:async?) and target.async?)
6
+ when true
7
+ Skein::Handler::Async.new(target)
8
+ else
9
+ Skein::Handler::Threaded.new(target)
10
+ end
11
+ end
12
+
13
+ # == Instance Methods =====================================================
14
+
15
+ def initialize(target)
16
+ @target = target
17
+ end
18
+
19
+ def handle(message_json)
20
+ # REFACTOR: Roll this into a module to keep it more contained.
21
+ # REFACTOR: Use Skein::RPC::Request
22
+ request =
23
+ begin
24
+ JSON.load(message_json)
25
+
26
+ rescue Object => e
27
+ @context.exception!(e, message_json)
28
+
29
+ return yield(JSON.dump(
30
+ result: nil,
31
+ error: '[%s] %s' % [ e.class, e ],
32
+ id: request['id']
33
+ ))
34
+ end
35
+
36
+ case (request)
37
+ when Hash
38
+ # Acceptable
39
+ else
40
+ return yield(JSON.dump(
41
+ result: nil,
42
+ error: 'Request does not conform to the JSON-RPC format.',
43
+ id: nil
44
+ ))
45
+ end
46
+
47
+ request['params'] =
48
+ case (params = request['params'])
49
+ when Array
50
+ params
51
+ when nil
52
+ request.key?('params') ? [ nil ] : [ ]
53
+ else
54
+ [ request['params'] ]
55
+ end
56
+
57
+ unless (request['method'] and request['method'].is_a?(String) and request['method'].match(/\S/))
58
+ return yield(JSON.dump(
59
+ result: nil,
60
+ error: 'Request does not conform to the JSON-RPC format, missing valid method.',
61
+ id: request['id']
62
+ ))
63
+ end
64
+
65
+ begin
66
+ delegate(request['method'], *request['params']) do |result, error = nil|
67
+ yield(JSON.dump(
68
+ result: result,
69
+ error: error,
70
+ id: request['id']
71
+ ))
72
+ end
73
+ rescue Object => e
74
+ @reporter and @reporter.exception!(e, message_json)
75
+
76
+ yield(JSON.dump(
77
+ result: nil,
78
+ error: '[%s] %s' % [ e.class, e ],
79
+ id: request['id']
80
+ ))
81
+ end
82
+ end
83
+ end
84
+
85
+ require_relative './handler/async'
86
+ require_relative './handler/threaded'