polyphony 0.50.1 → 0.51.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/CHANGELOG.md +5 -0
- data/Gemfile.lock +3 -1
- data/ext/polyphony/backend_common.h +7 -0
- data/ext/polyphony/backend_io_uring.c +93 -0
- data/ext/polyphony/backend_libev.c +54 -0
- data/lib/polyphony/adapters/process.rb +2 -2
- data/lib/polyphony/extensions/io.rb +4 -0
- data/lib/polyphony/extensions/socket.rb +12 -0
- data/lib/polyphony/version.rb +1 -1
- data/polyphony.gemspec +1 -0
- data/test/test_io.rb +53 -1
- data/test/test_socket.rb +34 -0
- metadata +16 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 72f11d867863c51f1fc1ab3ec85fe575cca9ee53a5047f4d3f7b76cea3b650b7
|
4
|
+
data.tar.gz: c29354b2de2f207185fc06fd86421b53d328aaf67d611109a6f95e9130ac5967
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 93351a3c4007145ff317257f82b764708b445e94b848d31f783f4d72071b097a8309322c4bedfdb7f321b0613ef436a314ad4e650928d0855143d3a5b98d9c13
|
7
|
+
data.tar.gz: 9119039548264867fa012bcbb90999f4ad82170c28539d43b0145687a761d508630da538ac87b75f9bf32f6b33cb0e13c22f50143248b85cc90289d5d3451d3f
|
data/CHANGELOG.md
CHANGED
data/Gemfile.lock
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
PATH
|
2
2
|
remote: .
|
3
3
|
specs:
|
4
|
-
polyphony (0.
|
4
|
+
polyphony (0.51.0)
|
5
5
|
|
6
6
|
GEM
|
7
7
|
remote: https://rubygems.org/
|
@@ -72,6 +72,7 @@ GEM
|
|
72
72
|
builder
|
73
73
|
minitest (>= 5.0)
|
74
74
|
ruby-progressbar
|
75
|
+
msgpack (1.4.2)
|
75
76
|
multi_xml (0.6.0)
|
76
77
|
mysql2 (0.5.3)
|
77
78
|
parallel (1.19.1)
|
@@ -136,6 +137,7 @@ DEPENDENCIES
|
|
136
137
|
just-the-docs (~> 0.3.0)
|
137
138
|
minitest (= 5.13.0)
|
138
139
|
minitest-reporters (= 1.4.2)
|
140
|
+
msgpack (= 1.4.2)
|
139
141
|
mysql2 (= 0.5.3)
|
140
142
|
pg (= 1.1.4)
|
141
143
|
polyphony!
|
@@ -107,6 +107,13 @@ inline VALUE backend_snooze() {
|
|
107
107
|
READ_LOOP_PREPARE_STR(); \
|
108
108
|
}
|
109
109
|
|
110
|
+
#define READ_LOOP_PASS_STR_TO_RECEIVER(receiver, method_id) { \
|
111
|
+
io_set_read_length(str, total, shrinkable); \
|
112
|
+
io_enc_str(str, fptr); \
|
113
|
+
rb_funcall_passing_block(receiver, method_id, 1, &str); \
|
114
|
+
READ_LOOP_PREPARE_STR(); \
|
115
|
+
}
|
116
|
+
|
110
117
|
inline void rectify_io_file_pos(rb_io_t *fptr) {
|
111
118
|
// Apparently after reopening a closed file, the file position is not reset,
|
112
119
|
// which causes the read to fail. Fortunately we can use fptr->rbuf.len to
|
@@ -389,6 +389,52 @@ VALUE Backend_read_loop(VALUE self, VALUE io) {
|
|
389
389
|
return io;
|
390
390
|
}
|
391
391
|
|
392
|
+
VALUE Backend_feed_loop(VALUE self, VALUE io, VALUE receiver, VALUE method) {
|
393
|
+
Backend_t *backend;
|
394
|
+
rb_io_t *fptr;
|
395
|
+
VALUE str;
|
396
|
+
long total;
|
397
|
+
long len = 8192;
|
398
|
+
int shrinkable;
|
399
|
+
char *buf;
|
400
|
+
VALUE underlying_io = rb_ivar_get(io, ID_ivar_io);
|
401
|
+
ID method_id = SYM2ID(method);
|
402
|
+
|
403
|
+
READ_LOOP_PREPARE_STR();
|
404
|
+
|
405
|
+
GetBackend(self, backend);
|
406
|
+
if (underlying_io != Qnil) io = underlying_io;
|
407
|
+
GetOpenFile(io, fptr);
|
408
|
+
rb_io_check_byte_readable(fptr);
|
409
|
+
rectify_io_file_pos(fptr);
|
410
|
+
|
411
|
+
while (1) {
|
412
|
+
VALUE resume_value = Qnil;
|
413
|
+
op_context_t *ctx = OP_CONTEXT_ACQUIRE(&backend->store, OP_READ);
|
414
|
+
struct io_uring_sqe *sqe = io_uring_get_sqe(&backend->ring);
|
415
|
+
io_uring_prep_read(sqe, fptr->fd, buf, len, -1);
|
416
|
+
|
417
|
+
ssize_t result = io_uring_backend_defer_submit_and_await(backend, sqe, ctx, &resume_value);
|
418
|
+
OP_CONTEXT_RELEASE(&backend->store, ctx);
|
419
|
+
RAISE_IF_EXCEPTION(resume_value);
|
420
|
+
if (!ctx->completed) return resume_value;
|
421
|
+
RB_GC_GUARD(resume_value);
|
422
|
+
|
423
|
+
if (result < 0)
|
424
|
+
rb_syserr_fail(-result, strerror(-result));
|
425
|
+
else if (!result)
|
426
|
+
break; // EOF
|
427
|
+
else {
|
428
|
+
total = result;
|
429
|
+
READ_LOOP_PASS_STR_TO_RECEIVER(receiver, method_id);
|
430
|
+
}
|
431
|
+
}
|
432
|
+
|
433
|
+
RB_GC_GUARD(str);
|
434
|
+
|
435
|
+
return io;
|
436
|
+
}
|
437
|
+
|
392
438
|
VALUE Backend_write(VALUE self, VALUE io, VALUE str) {
|
393
439
|
Backend_t *backend;
|
394
440
|
rb_io_t *fptr;
|
@@ -596,6 +642,51 @@ VALUE Backend_recv_loop(VALUE self, VALUE io) {
|
|
596
642
|
return io;
|
597
643
|
}
|
598
644
|
|
645
|
+
VALUE Backend_recv_feed_loop(VALUE self, VALUE io, VALUE receiver, VALUE method) {
|
646
|
+
Backend_t *backend;
|
647
|
+
rb_io_t *fptr;
|
648
|
+
VALUE str;
|
649
|
+
long total;
|
650
|
+
long len = 8192;
|
651
|
+
int shrinkable;
|
652
|
+
char *buf;
|
653
|
+
VALUE underlying_io = rb_ivar_get(io, ID_ivar_io);
|
654
|
+
ID method_id = SYM2ID(method);
|
655
|
+
|
656
|
+
READ_LOOP_PREPARE_STR();
|
657
|
+
|
658
|
+
GetBackend(self, backend);
|
659
|
+
if (underlying_io != Qnil) io = underlying_io;
|
660
|
+
GetOpenFile(io, fptr);
|
661
|
+
rb_io_check_byte_readable(fptr);
|
662
|
+
rectify_io_file_pos(fptr);
|
663
|
+
|
664
|
+
while (1) {
|
665
|
+
VALUE resume_value = Qnil;
|
666
|
+
op_context_t *ctx = OP_CONTEXT_ACQUIRE(&backend->store, OP_RECV);
|
667
|
+
struct io_uring_sqe *sqe = io_uring_get_sqe(&backend->ring);
|
668
|
+
io_uring_prep_recv(sqe, fptr->fd, buf, len, 0);
|
669
|
+
|
670
|
+
int result = io_uring_backend_defer_submit_and_await(backend, sqe, ctx, &resume_value);
|
671
|
+
OP_CONTEXT_RELEASE(&backend->store, ctx);
|
672
|
+
RAISE_IF_EXCEPTION(resume_value);
|
673
|
+
if (!ctx->completed) return resume_value;
|
674
|
+
RB_GC_GUARD(resume_value);
|
675
|
+
|
676
|
+
if (result < 0)
|
677
|
+
rb_syserr_fail(-result, strerror(-result));
|
678
|
+
else if (!result)
|
679
|
+
break; // EOF
|
680
|
+
else {
|
681
|
+
total = result;
|
682
|
+
READ_LOOP_PASS_STR_TO_RECEIVER(receiver, method_id);
|
683
|
+
}
|
684
|
+
}
|
685
|
+
|
686
|
+
RB_GC_GUARD(str);
|
687
|
+
return io;
|
688
|
+
}
|
689
|
+
|
599
690
|
VALUE Backend_send(VALUE self, VALUE io, VALUE str) {
|
600
691
|
Backend_t *backend;
|
601
692
|
rb_io_t *fptr;
|
@@ -918,9 +1009,11 @@ void Init_Backend() {
|
|
918
1009
|
|
919
1010
|
rb_define_method(cBackend, "read", Backend_read, 4);
|
920
1011
|
rb_define_method(cBackend, "read_loop", Backend_read_loop, 1);
|
1012
|
+
rb_define_method(cBackend, "feed_loop", Backend_feed_loop, 3);
|
921
1013
|
rb_define_method(cBackend, "write", Backend_write_m, -1);
|
922
1014
|
rb_define_method(cBackend, "recv", Backend_recv, 3);
|
923
1015
|
rb_define_method(cBackend, "recv_loop", Backend_recv_loop, 1);
|
1016
|
+
rb_define_method(cBackend, "recv_feed_loop", Backend_recv_feed_loop, 3);
|
924
1017
|
rb_define_method(cBackend, "send", Backend_send, 2);
|
925
1018
|
rb_define_method(cBackend, "accept", Backend_accept, 2);
|
926
1019
|
rb_define_method(cBackend, "accept_loop", Backend_accept_loop, 2);
|
@@ -334,6 +334,58 @@ error:
|
|
334
334
|
return RAISE_EXCEPTION(switchpoint_result);
|
335
335
|
}
|
336
336
|
|
337
|
+
VALUE Backend_feed_loop(VALUE self, VALUE io, VALUE receiver, VALUE method) {
|
338
|
+
Backend_t *backend;
|
339
|
+
struct libev_io watcher;
|
340
|
+
rb_io_t *fptr;
|
341
|
+
VALUE str;
|
342
|
+
long total;
|
343
|
+
long len = 8192;
|
344
|
+
int shrinkable;
|
345
|
+
char *buf;
|
346
|
+
VALUE switchpoint_result = Qnil;
|
347
|
+
VALUE underlying_io = rb_ivar_get(io, ID_ivar_io);
|
348
|
+
ID method_id = SYM2ID(method);
|
349
|
+
|
350
|
+
READ_LOOP_PREPARE_STR();
|
351
|
+
|
352
|
+
GetBackend(self, backend);
|
353
|
+
if (underlying_io != Qnil) io = underlying_io;
|
354
|
+
GetOpenFile(io, fptr);
|
355
|
+
rb_io_check_byte_readable(fptr);
|
356
|
+
io_set_nonblock(fptr, io);
|
357
|
+
rectify_io_file_pos(fptr);
|
358
|
+
watcher.fiber = Qnil;
|
359
|
+
|
360
|
+
while (1) {
|
361
|
+
ssize_t n = read(fptr->fd, buf, len);
|
362
|
+
if (n < 0) {
|
363
|
+
int e = errno;
|
364
|
+
if ((e != EWOULDBLOCK && e != EAGAIN)) rb_syserr_fail(e, strerror(e));
|
365
|
+
|
366
|
+
switchpoint_result = libev_wait_fd_with_watcher(backend, fptr->fd, &watcher, EV_READ);
|
367
|
+
if (TEST_EXCEPTION(switchpoint_result)) goto error;
|
368
|
+
}
|
369
|
+
else {
|
370
|
+
switchpoint_result = backend_snooze();
|
371
|
+
|
372
|
+
if (TEST_EXCEPTION(switchpoint_result)) goto error;
|
373
|
+
|
374
|
+
if (n == 0) break; // EOF
|
375
|
+
total = n;
|
376
|
+
READ_LOOP_PASS_STR_TO_RECEIVER(receiver, method_id);
|
377
|
+
}
|
378
|
+
}
|
379
|
+
|
380
|
+
RB_GC_GUARD(str);
|
381
|
+
RB_GC_GUARD(watcher.fiber);
|
382
|
+
RB_GC_GUARD(switchpoint_result);
|
383
|
+
|
384
|
+
return io;
|
385
|
+
error:
|
386
|
+
return RAISE_EXCEPTION(switchpoint_result);
|
387
|
+
}
|
388
|
+
|
337
389
|
VALUE Backend_write(VALUE self, VALUE io, VALUE str) {
|
338
390
|
Backend_t *backend;
|
339
391
|
struct libev_io watcher;
|
@@ -811,12 +863,14 @@ void Init_Backend() {
|
|
811
863
|
|
812
864
|
rb_define_method(cBackend, "read", Backend_read, 4);
|
813
865
|
rb_define_method(cBackend, "read_loop", Backend_read_loop, 1);
|
866
|
+
rb_define_method(cBackend, "feed_loop", Backend_feed_loop, 3);
|
814
867
|
rb_define_method(cBackend, "write", Backend_write_m, -1);
|
815
868
|
rb_define_method(cBackend, "accept", Backend_accept, 2);
|
816
869
|
rb_define_method(cBackend, "accept_loop", Backend_accept_loop, 2);
|
817
870
|
rb_define_method(cBackend, "connect", Backend_connect, 3);
|
818
871
|
rb_define_method(cBackend, "recv", Backend_recv, 3);
|
819
872
|
rb_define_method(cBackend, "recv_loop", Backend_read_loop, 1);
|
873
|
+
rb_define_method(cBackend, "recv_feed_loop", Backend_feed_loop, 3);
|
820
874
|
rb_define_method(cBackend, "send", Backend_write, 2);
|
821
875
|
rb_define_method(cBackend, "wait_io", Backend_wait_io, 2);
|
822
876
|
rb_define_method(cBackend, "sleep", Backend_sleep, 1);
|
@@ -220,6 +220,10 @@ class ::IO
|
|
220
220
|
Thread.current.backend.read_loop(self, &block)
|
221
221
|
end
|
222
222
|
|
223
|
+
def feed_loop(receiver, method, &block)
|
224
|
+
Thread.current.backend.feed_loop(self, receiver, method, &block)
|
225
|
+
end
|
226
|
+
|
223
227
|
# alias_method :orig_read, :read
|
224
228
|
# def read(length = nil, outbuf = nil)
|
225
229
|
# if length
|
@@ -31,6 +31,10 @@ class ::Socket
|
|
31
31
|
end
|
32
32
|
alias_method :read_loop, :recv_loop
|
33
33
|
|
34
|
+
def feed_loop(receiver, method, &block)
|
35
|
+
Thread.current.backend.recv_feed_loop(self, receiver, method, &block)
|
36
|
+
end
|
37
|
+
|
34
38
|
def recvfrom(maxlen, flags = 0)
|
35
39
|
@read_buffer ||= +''
|
36
40
|
while true
|
@@ -146,6 +150,10 @@ class ::TCPSocket
|
|
146
150
|
end
|
147
151
|
alias_method :read_loop, :recv_loop
|
148
152
|
|
153
|
+
def feed_loop(receiver, method, &block)
|
154
|
+
Thread.current.backend.recv_feed_loop(self, receiver, method, &block)
|
155
|
+
end
|
156
|
+
|
149
157
|
def send(mesg, flags = 0)
|
150
158
|
Thread.current.backend.send(self, mesg)
|
151
159
|
end
|
@@ -227,6 +235,10 @@ class ::UNIXSocket
|
|
227
235
|
end
|
228
236
|
alias_method :read_loop, :recv_loop
|
229
237
|
|
238
|
+
def feed_loop(receiver, method, &block)
|
239
|
+
Thread.current.backend.recv_feed_loop(self, receiver, method, &block)
|
240
|
+
end
|
241
|
+
|
230
242
|
def send(mesg, flags = 0)
|
231
243
|
Thread.current.backend.send(self, mesg)
|
232
244
|
end
|
data/lib/polyphony/version.rb
CHANGED
data/polyphony.gemspec
CHANGED
@@ -28,6 +28,7 @@ Gem::Specification.new do |s|
|
|
28
28
|
s.add_development_dependency 'rubocop', '0.85.1'
|
29
29
|
s.add_development_dependency 'pry', '0.13.1'
|
30
30
|
|
31
|
+
s.add_development_dependency 'msgpack', '1.4.2'
|
31
32
|
s.add_development_dependency 'pg', '1.1.4'
|
32
33
|
s.add_development_dependency 'redis', '4.1.0'
|
33
34
|
s.add_development_dependency 'hiredis', '0.6.3'
|
data/test/test_io.rb
CHANGED
@@ -1,6 +1,7 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
3
|
require_relative 'helper'
|
4
|
+
require 'msgpack'
|
4
5
|
|
5
6
|
class IOTest < MiniTest::Test
|
6
7
|
def setup
|
@@ -110,7 +111,7 @@ class IOTest < MiniTest::Test
|
|
110
111
|
assert_equal [], buf
|
111
112
|
|
112
113
|
o << "ulous\n"
|
113
|
-
|
114
|
+
sleep 0.01
|
114
115
|
assert_equal ["fabulous\n"], buf
|
115
116
|
|
116
117
|
o.close
|
@@ -173,6 +174,57 @@ class IOTest < MiniTest::Test
|
|
173
174
|
|
174
175
|
assert_equal 'hello: world', buf
|
175
176
|
end
|
177
|
+
|
178
|
+
def test_feed_loop_with_block
|
179
|
+
i, o = IO.pipe
|
180
|
+
unpacker = MessagePack::Unpacker.new
|
181
|
+
buffer = []
|
182
|
+
reader = spin do
|
183
|
+
i.feed_loop(unpacker, :feed_each) { |msg| buffer << msg }
|
184
|
+
end
|
185
|
+
o << 'foo'.to_msgpack
|
186
|
+
sleep 0.01
|
187
|
+
assert_equal ['foo'], buffer
|
188
|
+
|
189
|
+
o << 'bar'.to_msgpack
|
190
|
+
sleep 0.01
|
191
|
+
assert_equal ['foo', 'bar'], buffer
|
192
|
+
|
193
|
+
o << 'baz'.to_msgpack
|
194
|
+
sleep 0.01
|
195
|
+
assert_equal ['foo', 'bar', 'baz'], buffer
|
196
|
+
end
|
197
|
+
|
198
|
+
class Receiver
|
199
|
+
attr_reader :buffer
|
200
|
+
|
201
|
+
def initialize
|
202
|
+
@buffer = []
|
203
|
+
end
|
204
|
+
|
205
|
+
def recv(obj)
|
206
|
+
@buffer << obj
|
207
|
+
end
|
208
|
+
end
|
209
|
+
|
210
|
+
def test_feed_loop_without_block
|
211
|
+
i, o = IO.pipe
|
212
|
+
receiver = Receiver.new
|
213
|
+
reader = spin do
|
214
|
+
i.feed_loop(receiver, :recv)
|
215
|
+
end
|
216
|
+
o << 'foo'
|
217
|
+
sleep 0.01
|
218
|
+
assert_equal ['foo'], receiver.buffer
|
219
|
+
|
220
|
+
o << 'bar'
|
221
|
+
sleep 0.01
|
222
|
+
assert_equal ['foo', 'bar'], receiver.buffer
|
223
|
+
|
224
|
+
o << 'baz'
|
225
|
+
sleep 0.01
|
226
|
+
assert_equal ['foo', 'bar', 'baz'], receiver.buffer
|
227
|
+
end
|
176
228
|
end
|
177
229
|
|
178
230
|
class IOClassMethodsTest < MiniTest::Test
|
data/test/test_socket.rb
CHANGED
@@ -2,6 +2,7 @@
|
|
2
2
|
|
3
3
|
require_relative 'helper'
|
4
4
|
require 'fileutils'
|
5
|
+
require 'msgpack'
|
5
6
|
|
6
7
|
class SocketTest < MiniTest::Test
|
7
8
|
def setup
|
@@ -33,6 +34,39 @@ class SocketTest < MiniTest::Test
|
|
33
34
|
server&.close
|
34
35
|
end
|
35
36
|
|
37
|
+
def test_feed_loop
|
38
|
+
port = rand(1234..5678)
|
39
|
+
server = TCPServer.new('127.0.0.1', port)
|
40
|
+
|
41
|
+
server_fiber = spin do
|
42
|
+
reader = MessagePack::Unpacker.new
|
43
|
+
while (socket = server.accept)
|
44
|
+
spin do
|
45
|
+
socket.feed_loop(reader, :feed_each) do |msg|
|
46
|
+
msg = { 'result' => msg['x'] + msg['y'] }
|
47
|
+
socket << msg.to_msgpack
|
48
|
+
end
|
49
|
+
end
|
50
|
+
end
|
51
|
+
end
|
52
|
+
|
53
|
+
snooze
|
54
|
+
client = TCPSocket.new('127.0.0.1', port)
|
55
|
+
reader = MessagePack::Unpacker.new
|
56
|
+
client << { 'x' => 13, 'y' => 14 }.to_msgpack
|
57
|
+
result = nil
|
58
|
+
client.feed_loop(reader, :feed_each) do |msg|
|
59
|
+
result = msg
|
60
|
+
break
|
61
|
+
end
|
62
|
+
assert_equal({ 'result' => 27}, result)
|
63
|
+
client.close
|
64
|
+
ensure
|
65
|
+
server_fiber&.stop
|
66
|
+
server_fiber&.await
|
67
|
+
server&.close
|
68
|
+
end
|
69
|
+
|
36
70
|
def test_unix_socket
|
37
71
|
path = '/tmp/test_unix_socket'
|
38
72
|
FileUtils.rm(path) rescue nil
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: polyphony
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 0.51.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Sharon Rosner
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2021-
|
11
|
+
date: 2021-02-02 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: rake-compiler
|
@@ -94,6 +94,20 @@ dependencies:
|
|
94
94
|
- - '='
|
95
95
|
- !ruby/object:Gem::Version
|
96
96
|
version: 0.13.1
|
97
|
+
- !ruby/object:Gem::Dependency
|
98
|
+
name: msgpack
|
99
|
+
requirement: !ruby/object:Gem::Requirement
|
100
|
+
requirements:
|
101
|
+
- - '='
|
102
|
+
- !ruby/object:Gem::Version
|
103
|
+
version: 1.4.2
|
104
|
+
type: :development
|
105
|
+
prerelease: false
|
106
|
+
version_requirements: !ruby/object:Gem::Requirement
|
107
|
+
requirements:
|
108
|
+
- - '='
|
109
|
+
- !ruby/object:Gem::Version
|
110
|
+
version: 1.4.2
|
97
111
|
- !ruby/object:Gem::Dependency
|
98
112
|
name: pg
|
99
113
|
requirement: !ruby/object:Gem::Requirement
|