protobuf 1.1.3 → 1.3.0

Sign up to get free protection for your applications and to get access to all the features.
Files changed (60) hide show
  1. data/.gitignore +3 -0
  2. data/Gemfile.lock +44 -25
  3. data/README.md +2 -2
  4. data/Rakefile +15 -0
  5. data/bin/rpc_server +52 -11
  6. data/lib/protobuf.rb +22 -10
  7. data/lib/protobuf/common/logger.rb +26 -25
  8. data/lib/protobuf/descriptor/file_descriptor.rb +1 -1
  9. data/lib/protobuf/message/field.rb +2 -2
  10. data/lib/protobuf/rpc/buffer.rb +30 -25
  11. data/lib/protobuf/rpc/client.rb +8 -8
  12. data/lib/protobuf/rpc/connector.rb +2 -0
  13. data/lib/protobuf/rpc/connectors/base.rb +0 -1
  14. data/lib/protobuf/rpc/connectors/common.rb +48 -48
  15. data/lib/protobuf/rpc/connectors/em_client.rb +53 -27
  16. data/lib/protobuf/rpc/connectors/eventmachine.rb +14 -17
  17. data/lib/protobuf/rpc/connectors/socket.rb +23 -16
  18. data/lib/protobuf/rpc/connectors/zmq.rb +73 -0
  19. data/lib/protobuf/rpc/error.rb +1 -2
  20. data/lib/protobuf/rpc/error/client_error.rb +4 -4
  21. data/lib/protobuf/rpc/server.rb +31 -43
  22. data/lib/protobuf/rpc/servers/evented/server.rb +43 -0
  23. data/lib/protobuf/rpc/servers/evented_runner.rb +1 -1
  24. data/lib/protobuf/rpc/servers/socket/server.rb +108 -0
  25. data/lib/protobuf/rpc/servers/socket/worker.rb +59 -0
  26. data/lib/protobuf/rpc/servers/socket_runner.rb +3 -3
  27. data/lib/protobuf/rpc/servers/zmq/broker.rb +85 -0
  28. data/lib/protobuf/rpc/servers/zmq/server.rb +50 -0
  29. data/lib/protobuf/rpc/servers/zmq/util.rb +27 -0
  30. data/lib/protobuf/rpc/servers/zmq/worker.rb +72 -0
  31. data/lib/protobuf/rpc/servers/zmq_runner.rb +26 -0
  32. data/lib/protobuf/rpc/service.rb +5 -5
  33. data/lib/protobuf/version.rb +1 -1
  34. data/protobuf.gemspec +12 -10
  35. data/spec/benchmark/tasks.rb +37 -5
  36. data/spec/functional/evented_server_spec.rb +64 -0
  37. data/spec/functional/socket_server_spec.rb +63 -0
  38. data/spec/functional/zmq_server_spec.rb +63 -0
  39. data/spec/helper/server.rb +32 -12
  40. data/spec/lib/protobuf/message/encoder_spec.rb +19 -0
  41. data/spec/proto/test.pb.rb +3 -3
  42. data/spec/proto/test.proto +3 -3
  43. data/spec/proto/test_service.rb +1 -0
  44. data/spec/spec_helper.rb +6 -0
  45. data/spec/unit/message_spec.rb +1 -1
  46. data/spec/unit/rpc/client_spec.rb +11 -3
  47. data/spec/unit/rpc/connectors/common_spec.rb +0 -1
  48. data/spec/unit/rpc/connectors/eventmachine_client_spec.rb +32 -0
  49. data/spec/unit/rpc/connectors/socket_spec.rb +2 -4
  50. data/spec/unit/rpc/connectors/zmq_spec.rb +27 -0
  51. data/spec/unit/rpc/servers/evented_server_spec.rb +3 -3
  52. data/spec/unit/rpc/servers/socket_server_spec.rb +14 -13
  53. data/spec/unit/rpc/servers/zmq/broker_spec.rb +27 -0
  54. data/spec/unit/rpc/servers/zmq/server_spec.rb +37 -0
  55. data/spec/unit/rpc/servers/zmq/util_spec.rb +41 -0
  56. data/spec/unit/rpc/servers/zmq/worker_spec.rb +36 -0
  57. data/spec/unit/rpc/service_spec.rb +22 -18
  58. metadata +87 -40
  59. data/lib/protobuf/rpc/servers/evented_server.rb +0 -28
  60. data/lib/protobuf/rpc/servers/socket_server.rb +0 -146
@@ -1,28 +0,0 @@
1
- require 'protobuf/rpc/server'
2
-
3
- module Protobuf
4
- module Rpc
5
- class EventedServer < EventMachine::Connection
6
- include Protobuf::Rpc::Server
7
- include Protobuf::Logger::LogMethods
8
-
9
- # Initialize a new read buffer for storing client request info
10
- def post_init
11
- log_debug '[server] Post init, new read buffer created'
12
- @stats = Protobuf::Rpc::Stat.new(:SERVER, true)
13
- @stats.client = Socket.unpack_sockaddr_in(get_peername)
14
-
15
- @buffer = Protobuf::Rpc::Buffer.new(:read)
16
- @did_respond = false
17
- end
18
-
19
- # Receive a chunk of data, potentially flushed to handle_client
20
- def receive_data(data)
21
- log_debug '[server] receive_data: %s' % data
22
- @buffer << data
23
- handle_client if @buffer.flushed?
24
- end
25
-
26
- end
27
- end
28
- end
@@ -1,146 +0,0 @@
1
- require 'protobuf/rpc/server'
2
-
3
- module Protobuf
4
- module Rpc
5
- class SocketServer
6
- include Protobuf::Rpc::Server
7
- include Protobuf::Logger::LogMethods
8
-
9
-
10
- def self.cleanup?
11
- # every 10 connections run a cleanup routine after closing the response
12
- @threads.size > (@thread_threshold - 1) && (@threads.size % @thread_threshold) == 0
13
- end
14
-
15
- def self.cleanup_threads
16
- log_debug "[#{log_signature}] Thread cleanup - #{@threads.size} - start"
17
-
18
- @threads = @threads.select do |t|
19
- if t[:thread].alive?
20
- true
21
- else
22
- t[:thread].join
23
- @working.delete(t[:socket])
24
- false
25
- end
26
- end
27
-
28
- log_debug "[#{log_signature}] Thread cleanup - #{@threads.size} - complete"
29
- end
30
-
31
- def self.log_signature
32
- @log_signature ||= "server-#{self}"
33
- end
34
-
35
- def self.new_worker(socket)
36
- Thread.new(socket) do |sock|
37
- Protobuf::Rpc::SocketServer::Worker.new(sock) do |s|
38
- s.close
39
- end
40
- end
41
- end
42
-
43
- def self.run(opts = {})
44
- log_debug "[#{log_signature}] Run"
45
- host = opts.fetch(:host, "127.0.0.1")
46
- port = opts.fetch(:port, 9399)
47
- backlog = opts.fetch(:backlog, 100)
48
- thread_threshold = opts.fetch(:thread_threshold, 100)
49
- auto_collect_timeout = opts.fetch(:auto_collect_timeout, 20)
50
-
51
- @running = true
52
- @threads = []
53
- @thread_threshold = thread_threshold
54
- @server = TCPServer.new(host, port)
55
- @server.listen(backlog)
56
- @working = []
57
- @listen_fds = [@server]
58
-
59
- while running?
60
- log_debug "[#{log_signature}] Waiting for connections"
61
-
62
- if ready_cnxns = IO.select(@listen_fds, [], [], auto_collect_timeout)
63
- cnxns = ready_cnxns.first
64
- cnxns.each do |client|
65
- case
66
- when !running? then
67
- # no-op
68
- when client == @server then
69
- log_debug "[#{log_signature}] Accepted new connection"
70
- client, sockaddr = @server.accept
71
- @listen_fds << client
72
- else
73
- if !@working.include?(client)
74
- @working << @listen_fds.delete(client)
75
- log_debug "[#{log_signature}] Working"
76
- @threads << { :thread => new_worker(client), :socket => client }
77
-
78
- cleanup_threads if cleanup?
79
- end
80
- end
81
- end
82
- else
83
- # Run a cleanup if select times out while waiting
84
- cleanup_threads if @threads.size > 1
85
- end
86
- end
87
-
88
- rescue
89
- # Closing the server causes the loop to raise an exception here
90
- raise if running?
91
- end
92
-
93
- def self.running?
94
- @running
95
- end
96
-
97
- def self.stop
98
- @running = false
99
- @server.close if @server
100
- end
101
-
102
- class Worker
103
- include Protobuf::Rpc::Server
104
- include Protobuf::Logger::LogMethods
105
-
106
- def initialize(sock, &complete_cb)
107
- @did_response = false
108
- @socket = sock
109
- @request = Protobuf::Socketrpc::Request.new
110
- @response = Protobuf::Socketrpc::Response.new
111
- @buffer = Protobuf::Rpc::Buffer.new(:read)
112
- @stats = Protobuf::Rpc::Stat.new(:SERVER, true)
113
- @complete_cb = complete_cb
114
- log_debug "[#{log_signature}] Post init, new read buffer created"
115
-
116
- @stats.client = Socket.unpack_sockaddr_in(@socket.getpeername)
117
- @buffer << read_data
118
- log_debug "[#{log_signature}] handling request"
119
- handle_client if @buffer.flushed?
120
- end
121
-
122
- def log_signature
123
- @log_signature ||= "server-#{self.class}-#{object_id}"
124
- end
125
-
126
- def read_data
127
- size_io = StringIO.new
128
-
129
- while((size_reader = @socket.getc) != "-")
130
- size_io << size_reader
131
- end
132
- str_size_io = size_io.string
133
-
134
- "#{str_size_io}-#{@socket.read(str_size_io.to_i)}"
135
- end
136
-
137
- def send_data(data)
138
- log_debug "[#{log_signature}] sending data : %s" % data
139
- @socket.write(data)
140
- @socket.flush
141
- @complete_cb.call(@socket)
142
- end
143
- end
144
- end
145
- end
146
- end