packet 0.1.2 → 0.1.3
Sign up to get free protection for your applications and to get access to all the features.
- data/Rakefile +1 -1
- data/TODO +2 -0
- data/examples/netbeans.jpg +0 -0
- data/examples/write_bulk.rb +41 -0
- data/lib/packet.rb +19 -26
- data/lib/{disconnect_error.rb → packet/disconnect_error.rb} +3 -2
- data/lib/{double_keyed_hash.rb → packet/double_keyed_hash.rb} +0 -0
- data/lib/{callback.rb → packet/packet_callback.rb} +0 -0
- data/lib/{connection.rb → packet/packet_connection.rb} +10 -3
- data/lib/{core.rb → packet/packet_core.rb} +84 -40
- data/lib/{event.rb → packet/packet_event.rb} +1 -1
- data/lib/{packet_guid.rb → packet/packet_guid.rb} +0 -0
- data/lib/{class_helpers.rb → packet/packet_helper.rb} +0 -0
- data/lib/{packet_master.rb → packet/packet_master.rb} +18 -10
- data/lib/{meta_pimp.rb → packet/packet_meta_pimp.rb} +1 -1
- data/lib/packet/packet_mongrel.rb +218 -0
- data/lib/{nbio.rb → packet/packet_nbio.rb} +37 -37
- data/lib/{bin_parser.rb → packet/packet_parser.rb} +0 -0
- data/lib/{periodic_event.rb → packet/packet_periodic_event.rb} +0 -0
- data/lib/{pimp.rb → packet/packet_pimp.rb} +3 -1
- data/lib/{worker.rb → packet/packet_worker.rb} +1 -4
- data/lib/{timer_store.rb → packet/timer_store.rb} +26 -18
- data/lib/packet_mongrel.rb +200 -206
- data/spec/spec_helper.rb +10 -0
- data/spec/test_double_keyed_hash.rb +14 -0
- data/spec/test_packet_core.rb +38 -0
- metadata +25 -20
- data/lib/thread_pool.rb +0 -54
- data/lib/worker_pool.rb +0 -10
data/Rakefile
CHANGED
data/TODO
CHANGED
@@ -3,6 +3,8 @@
|
|
3
3
|
** TODO Remove class and class instance attributes from core classes.
|
4
4
|
** TODO Cleanup callback mechanism
|
5
5
|
** TODO Implement a sample worker, which provides capabilities to execute mysql statements async.
|
6
|
+
** TODO Implement more cleaner write thingies.
|
7
|
+
** TODO Implement more cleaner timers.
|
6
8
|
|
7
9
|
|
8
10
|
|
Binary file
|
@@ -0,0 +1,41 @@
|
|
1
|
+
require "socket"
|
2
|
+
require "thread"
|
3
|
+
|
4
|
+
# sock = TCPSocket.open("localhost",11007)
|
5
|
+
#data = File.open("netbeans.jpg").read
|
6
|
+
data = File.open("nginx.dat").read
|
7
|
+
# p data.length
|
8
|
+
|
9
|
+
threads = []
|
10
|
+
500.times do
|
11
|
+
# sock.write(data)
|
12
|
+
# select([sock],nil,nil,nil)
|
13
|
+
# read_data = ""
|
14
|
+
|
15
|
+
# loop do
|
16
|
+
# begin
|
17
|
+
# while(read_data << sock.read_nonblock(1023)); end
|
18
|
+
# rescue Errno::EAGAIN
|
19
|
+
# break
|
20
|
+
# rescue
|
21
|
+
# break
|
22
|
+
# end
|
23
|
+
# end
|
24
|
+
|
25
|
+
threads << Thread.new do
|
26
|
+
sock = TCPSocket.open("localhost",11007)
|
27
|
+
# p read_data.length
|
28
|
+
written_length = sock.write(data)
|
29
|
+
p "Write Length: #{written_length}"
|
30
|
+
read_length = sock.read(written_length)
|
31
|
+
p "Read length: #{read_length.length}"
|
32
|
+
end
|
33
|
+
|
34
|
+
# # p read_data.length
|
35
|
+
# written_length = sock.write(data)
|
36
|
+
# #p "Write Length: #{written_length}"
|
37
|
+
# read_length = sock.read(written_length)
|
38
|
+
# #p "Read length: #{read_length.length}"
|
39
|
+
end
|
40
|
+
|
41
|
+
threads.each { |x| x.join }
|
data/lib/packet.rb
CHANGED
@@ -1,37 +1,30 @@
|
|
1
|
+
$:.unshift(File.dirname(__FILE__)) unless
|
2
|
+
$:.include?(File.dirname(__FILE__)) || $:.include?(File.expand_path(File.dirname(__FILE__)))
|
3
|
+
|
1
4
|
require "socket"
|
2
5
|
require "yaml"
|
3
6
|
require "forwardable"
|
4
7
|
require "ostruct"
|
5
8
|
require "thread"
|
6
9
|
|
7
|
-
require "
|
8
|
-
|
9
|
-
|
10
|
-
require "
|
11
|
-
require "
|
12
|
-
require "
|
13
|
-
require "
|
14
|
-
require "
|
15
|
-
|
16
|
-
require "
|
17
|
-
require "
|
18
|
-
require "
|
19
|
-
|
20
|
-
require "
|
21
|
-
require "
|
22
|
-
require "meta_pimp"
|
23
|
-
require "core"
|
24
|
-
|
25
|
-
require "packet_master"
|
26
|
-
require "connection"
|
27
|
-
require "worker"
|
28
|
-
|
29
|
-
# This file is just a runner of things and hence does basic initialization of thingies required for running
|
30
|
-
# the application.
|
31
|
-
|
10
|
+
require "packet/packet_parser"
|
11
|
+
require "packet/packet_guid"
|
12
|
+
require "packet/packet_helper"
|
13
|
+
require "packet/double_keyed_hash"
|
14
|
+
require "packet/packet_event"
|
15
|
+
require "packet/packet_periodic_event"
|
16
|
+
require "packet/disconnect_error"
|
17
|
+
require "packet/packet_callback"
|
18
|
+
require "packet/packet_nbio"
|
19
|
+
require "packet/packet_pimp"
|
20
|
+
require "packet/packet_meta_pimp"
|
21
|
+
require "packet/packet_core"
|
22
|
+
require "packet/packet_master"
|
23
|
+
require "packet/packet_connection"
|
24
|
+
require "packet/packet_worker"
|
32
25
|
|
33
26
|
PACKET_APP = File.expand_path'../' unless defined?(PACKET_APP)
|
34
27
|
|
35
28
|
module Packet
|
36
|
-
VERSION='0.1.
|
29
|
+
VERSION='0.1.3'
|
37
30
|
end
|
File without changes
|
File without changes
|
@@ -3,21 +3,27 @@
|
|
3
3
|
|
4
4
|
module Packet
|
5
5
|
module Connection
|
6
|
+
attr_accessor :outbound_data,:connection_live
|
7
|
+
|
6
8
|
def send_data p_data
|
9
|
+
@outbound_data << p_data
|
7
10
|
begin
|
8
|
-
|
9
|
-
rescue DisconnectError =>
|
11
|
+
write_and_schedule(connection)
|
12
|
+
rescue DisconnectError => sock
|
10
13
|
close_connection
|
11
14
|
end
|
12
15
|
end
|
13
16
|
|
14
17
|
def invoke_init
|
15
18
|
@initialized = true
|
19
|
+
@connection_live = true
|
20
|
+
@outbound_data = []
|
16
21
|
post_init if respond_to?(:post_init)
|
17
22
|
end
|
18
23
|
|
19
|
-
def close_connection
|
24
|
+
def close_connection(sock = nil)
|
20
25
|
unbind if respond_to?(:unbind)
|
26
|
+
reactor.cancel_write(connection)
|
21
27
|
reactor.remove_connection(connection)
|
22
28
|
end
|
23
29
|
|
@@ -29,5 +35,6 @@ module Packet
|
|
29
35
|
def send_object p_object
|
30
36
|
dump_object(p_object,connection)
|
31
37
|
end
|
38
|
+
|
32
39
|
end # end of class Connection
|
33
40
|
end # end of module Packet
|
@@ -6,11 +6,11 @@ module Packet
|
|
6
6
|
base_klass.instance_eval do
|
7
7
|
@@connection_callbacks ||= {}
|
8
8
|
|
9
|
-
iattr_accessor :thread_pool_size
|
10
9
|
cattr_accessor :connection_callbacks
|
11
10
|
attr_accessor :read_ios, :write_ios, :listen_sockets
|
12
|
-
attr_accessor :connection_completion_awaited
|
13
|
-
attr_accessor :connections, :
|
11
|
+
attr_accessor :connection_completion_awaited,:write_scheduled
|
12
|
+
attr_accessor :connections, :windows_flag
|
13
|
+
attr_accessor :internal_scheduled_write,:outbound_data,:reactor
|
14
14
|
include CommonMethods
|
15
15
|
end
|
16
16
|
end
|
@@ -70,7 +70,6 @@ module Packet
|
|
70
70
|
client_socket,client_sockaddr = sock_io.accept_nonblock
|
71
71
|
client_socket.setsockopt(Socket::IPPROTO_TCP,Socket::TCP_NODELAY,1)
|
72
72
|
rescue Errno::EAGAIN, Errno::ECONNABORTED, Errno::EPROTO, Errno::EINTR
|
73
|
-
puts "not ready yet"
|
74
73
|
return
|
75
74
|
end
|
76
75
|
read_ios << client_socket
|
@@ -96,18 +95,12 @@ module Packet
|
|
96
95
|
|
97
96
|
# method removes the connection and closes the socket
|
98
97
|
def remove_connection(t_sock)
|
99
|
-
|
100
|
-
|
101
|
-
connections.delete(t_sock.fileno)
|
102
|
-
t_sock.close
|
103
|
-
end
|
104
|
-
|
105
|
-
def socket_really_connected?(t_sock)
|
98
|
+
read_ios.delete(t_sock)
|
99
|
+
write_ios.delete(t_sock)
|
106
100
|
begin
|
107
|
-
|
108
|
-
|
109
|
-
rescue
|
110
|
-
return false
|
101
|
+
connections.delete(t_sock.fileno)
|
102
|
+
t_sock.close
|
103
|
+
rescue
|
111
104
|
end
|
112
105
|
end
|
113
106
|
|
@@ -126,34 +119,70 @@ module Packet
|
|
126
119
|
Signal.trap("INT") { shutdown }
|
127
120
|
loop do
|
128
121
|
check_for_timer_events
|
129
|
-
|
130
|
-
|
131
|
-
|
132
|
-
|
133
|
-
|
134
|
-
|
135
|
-
|
136
|
-
|
137
|
-
|
138
|
-
|
139
|
-
|
140
|
-
|
141
|
-
|
122
|
+
ready_read_fds,ready_write_fds,read_error_fds = select(read_ios,write_ios,nil,0.005)
|
123
|
+
|
124
|
+
if ready_read_fds && !ready_read_fds.empty?
|
125
|
+
handle_read_event(ready_read_fds)
|
126
|
+
elsif ready_write_fds && !ready_write_fds.empty?
|
127
|
+
handle_write_event(ready_write_fds)
|
128
|
+
end
|
129
|
+
end
|
130
|
+
|
131
|
+
end
|
132
|
+
|
133
|
+
def schedule_write(t_sock)
|
134
|
+
fileno = t_sock.fileno
|
135
|
+
if UNIXSocket === t_sock && internal_scheduled_write[fileno].nil?
|
136
|
+
write_ios << t_sock
|
137
|
+
internal_scheduled_write[t_sock.fileno] ||= self
|
138
|
+
elsif write_scheduled[fileno].nil?
|
139
|
+
write_ios << t_sock
|
140
|
+
write_scheduled[fileno] ||= connections[fileno].instance
|
141
|
+
end
|
142
|
+
end
|
143
|
+
|
144
|
+
def cancel_write(t_sock)
|
145
|
+
fileno = t_sock.fileno
|
146
|
+
if UNIXSocket === t_sock
|
147
|
+
internal_scheduled_write.delete(fileno)
|
148
|
+
else
|
149
|
+
write_scheduled.delete(fileno)
|
150
|
+
end
|
151
|
+
write_ios.delete(t_sock)
|
152
|
+
end
|
153
|
+
|
154
|
+
def handle_write_event(p_ready_fds)
|
155
|
+
p_ready_fds.each do |sock_fd|
|
156
|
+
fileno = sock_fd.fileno
|
157
|
+
if UNIXSocket == sock_fd && internal_scheduled_write[fileno]
|
158
|
+
write_and_schedule(sock_fd)
|
159
|
+
elsif extern_opts = connection_completion_awaited[fileno]
|
160
|
+
complete_connection(sock_fd,extern_opts)
|
161
|
+
elsif handler_instance = write_scheduled[fileno]
|
162
|
+
handler_instance.write_scheduled(sock_fd)
|
142
163
|
end
|
143
164
|
end
|
144
165
|
end
|
145
166
|
|
146
|
-
def
|
147
|
-
|
148
|
-
|
167
|
+
def handle_read_event(p_ready_fds)
|
168
|
+
ready_fds = p_ready_fds.flatten.compact
|
169
|
+
ready_fds.each do |t_sock|
|
170
|
+
if(unix? && t_sock.is_a?(UNIXSocket))
|
171
|
+
handle_internal_messages(t_sock)
|
172
|
+
else
|
173
|
+
handle_external_messages(t_sock)
|
174
|
+
end
|
175
|
+
end
|
149
176
|
end
|
150
177
|
|
151
178
|
def terminate_me
|
152
179
|
# FIXME: close the open sockets
|
180
|
+
# @thread_pool.kill_all
|
153
181
|
exit
|
154
182
|
end
|
155
183
|
|
156
184
|
def shutdown
|
185
|
+
# @thread_pool.kill_all
|
157
186
|
# FIXME: close the open sockets
|
158
187
|
exit
|
159
188
|
end
|
@@ -166,8 +195,6 @@ module Packet
|
|
166
195
|
sock_fd = t_sock.fileno
|
167
196
|
if sock_opts = listen_sockets[sock_fd]
|
168
197
|
accept_connection(sock_opts)
|
169
|
-
elsif extern_opts = connection_completion_awaited[sock_fd]
|
170
|
-
complete_connection(t_sock,extern_opts)
|
171
198
|
else
|
172
199
|
read_external_socket(t_sock)
|
173
200
|
end
|
@@ -179,9 +206,7 @@ module Packet
|
|
179
206
|
t_data = read_data(t_sock)
|
180
207
|
handler_instance.receive_data(t_data) if handler_instance.respond_to?(:receive_data)
|
181
208
|
rescue DisconnectError => sock_error
|
182
|
-
handler_instance.
|
183
|
-
connections.delete(t_sock.fileno)
|
184
|
-
read_ios.delete(t_sock)
|
209
|
+
handler_instance.close_connection
|
185
210
|
end
|
186
211
|
end
|
187
212
|
|
@@ -202,17 +227,28 @@ module Packet
|
|
202
227
|
@timer_hash.delete(t_timer.timer_signature)
|
203
228
|
end
|
204
229
|
|
230
|
+
def binding_str
|
231
|
+
@binding += 1
|
232
|
+
"BIND_#{@binding}"
|
233
|
+
end
|
234
|
+
|
205
235
|
def initialize
|
206
236
|
@read_ios ||= []
|
207
237
|
@write_ios ||= []
|
208
238
|
@connection_completion_awaited ||= {}
|
239
|
+
@write_scheduled ||= {}
|
240
|
+
@internal_scheduled_write ||= {}
|
241
|
+
# internal outbound data
|
242
|
+
@outbound_data = []
|
209
243
|
@connections ||= {}
|
210
244
|
@listen_sockets ||= {}
|
245
|
+
@binding = 0
|
211
246
|
|
212
247
|
# @timer_hash = Packet::TimerStore
|
213
248
|
@timer_hash ||= {}
|
214
|
-
@thread_pool = ThreadPool.new(thread_pool_size || 20)
|
249
|
+
# @thread_pool = ThreadPool.new(thread_pool_size || 20)
|
215
250
|
@windows_flag = windows?
|
251
|
+
@reactor = self
|
216
252
|
end
|
217
253
|
|
218
254
|
def windows?
|
@@ -233,6 +269,15 @@ module Packet
|
|
233
269
|
end
|
234
270
|
end
|
235
271
|
end
|
272
|
+
|
273
|
+
# close the connection with internal specified socket
|
274
|
+
def close_connection(sock = nil)
|
275
|
+
begin
|
276
|
+
read_ios.delete(sock.fileno)
|
277
|
+
write_ios.delete(sock.fileno)
|
278
|
+
sock.close
|
279
|
+
rescue; end
|
280
|
+
end
|
236
281
|
|
237
282
|
def initialize_handler(p_module)
|
238
283
|
return p_module if(!p_module.is_a?(Class) and !p_module.is_a?(Module))
|
@@ -253,10 +298,9 @@ module Packet
|
|
253
298
|
handler_instance.unbind if handler_instance.respond_to?(:unbind)
|
254
299
|
return
|
255
300
|
end
|
256
|
-
|
257
|
-
handler_instance.signature = t_signature
|
301
|
+
handler_instance.signature = binding_str
|
258
302
|
connections[t_socket.fileno] =
|
259
|
-
OpenStruct.new(:socket => t_socket, :instance => handler_instance, :signature =>
|
303
|
+
OpenStruct.new(:socket => t_socket, :instance => handler_instance, :signature => handler_instance.signature,:sock_addr => sock_addr)
|
260
304
|
block.call(handler_instance) if block
|
261
305
|
handler_instance.connection_completed if handler_instance.respond_to?(:connection_completed)
|
262
306
|
end
|
File without changes
|
File without changes
|
@@ -14,7 +14,7 @@ module Packet
|
|
14
14
|
|
15
15
|
def self.run
|
16
16
|
master_reactor_instance = new
|
17
|
-
|
17
|
+
master_reactor_instance.result_hash = {}
|
18
18
|
master_reactor_instance.live_workers = DoubleKeyedHash.new
|
19
19
|
yield(master_reactor_instance)
|
20
20
|
master_reactor_instance.load_workers
|
@@ -34,7 +34,6 @@ module Packet
|
|
34
34
|
class << handler_instance
|
35
35
|
extend Forwardable
|
36
36
|
attr_accessor :workers,:connection,:reactor, :initialized,:signature
|
37
|
-
attr_accessor :thread_pool
|
38
37
|
include NbioHelper
|
39
38
|
include Connection
|
40
39
|
def ask_worker(*args)
|
@@ -52,7 +51,6 @@ module Packet
|
|
52
51
|
handler_instance.workers = @live_workers
|
53
52
|
handler_instance.connection = t_sock
|
54
53
|
handler_instance.reactor = self
|
55
|
-
handler_instance.thread_pool = @thread_pool
|
56
54
|
end
|
57
55
|
|
58
56
|
# FIXME: right now, each worker is tied to its connection and this can be problematic
|
@@ -65,10 +63,16 @@ module Packet
|
|
65
63
|
# t_data = Marshal.load(raw_data)
|
66
64
|
worker_instance.receive_data(raw_data) if worker_instance.respond_to?(:receive_data)
|
67
65
|
rescue DisconnectError => sock_error
|
68
|
-
|
66
|
+
remove_worker(t_sock)
|
69
67
|
end
|
70
68
|
end
|
69
|
+
|
71
70
|
|
71
|
+
def remove_worker(t_sock)
|
72
|
+
@live_workers.delete(t_sock.fileno)
|
73
|
+
read_ios.delete(t_sock)
|
74
|
+
end
|
75
|
+
|
72
76
|
def delete_worker(worker_options = {})
|
73
77
|
worker_name = worker_options[:worker]
|
74
78
|
worker_name_key = gen_worker_key(worker_name,worker_options[:job_key])
|
@@ -100,10 +104,14 @@ module Packet
|
|
100
104
|
worker_options.delete(:worker)
|
101
105
|
begin
|
102
106
|
require worker_name
|
107
|
+
worker_klass = Object.const_get(packet_classify(worker_name))
|
108
|
+
fork_and_load(worker_klass,worker_options)
|
103
109
|
rescue LoadError
|
110
|
+
puts "no such worker #{worker_name}"
|
111
|
+
rescue MissingSourceFile
|
112
|
+
puts "no such worker #{worker_name}"
|
113
|
+
return
|
104
114
|
end
|
105
|
-
worker_klass = Object.const_get(packet_classify(worker_name))
|
106
|
-
fork_and_load(worker_klass,worker_options)
|
107
115
|
end
|
108
116
|
|
109
117
|
# method forks given worker file in a new process
|
@@ -116,14 +124,14 @@ module Packet
|
|
116
124
|
master_read_end,worker_write_end = UNIXSocket.pair(Socket::SOCK_STREAM)
|
117
125
|
# socket to which master process is going to write
|
118
126
|
worker_read_end,master_write_end = UNIXSocket.pair(Socket::SOCK_STREAM)
|
119
|
-
worker_read_fd,master_write_fd = UNIXSocket.pair
|
127
|
+
# worker_read_fd,master_write_fd = UNIXSocket.pair
|
120
128
|
|
121
129
|
if((pid = fork()).nil?)
|
122
130
|
$0 = "ruby #{worker_klass.worker_name}"
|
123
|
-
[master_write_end,master_read_end
|
131
|
+
[master_write_end,master_read_end].each { |x| x.close }
|
124
132
|
|
125
133
|
worker_klass.start_worker(:write_end => worker_write_end,:read_end => worker_read_end,\
|
126
|
-
:
|
134
|
+
:options => worker_options)
|
127
135
|
end
|
128
136
|
Process.detach(pid)
|
129
137
|
|
@@ -136,12 +144,12 @@ module Packet
|
|
136
144
|
else
|
137
145
|
t_pimp = Packet::MetaPimp.new(master_write_end,pid,self)
|
138
146
|
t_pimp.worker_key = worker_name_key
|
147
|
+
t_pimp.worker_name = t_worker_name
|
139
148
|
@live_workers[worker_name_key,master_read_end.fileno] = t_pimp
|
140
149
|
end
|
141
150
|
|
142
151
|
worker_read_end.close
|
143
152
|
worker_write_end.close
|
144
|
-
worker_read_fd.close
|
145
153
|
read_ios << master_read_end
|
146
154
|
end # end of fork_and_load method
|
147
155
|
end # end of Reactor class
|