polyphony 0.50.1 → 0.51.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +5 -0
- data/Gemfile.lock +3 -1
- data/ext/polyphony/backend_common.h +7 -0
- data/ext/polyphony/backend_io_uring.c +93 -0
- data/ext/polyphony/backend_libev.c +54 -0
- data/lib/polyphony/adapters/process.rb +2 -2
- data/lib/polyphony/extensions/io.rb +4 -0
- data/lib/polyphony/extensions/socket.rb +12 -0
- data/lib/polyphony/version.rb +1 -1
- data/polyphony.gemspec +1 -0
- data/test/test_io.rb +53 -1
- data/test/test_socket.rb +34 -0
- metadata +16 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 72f11d867863c51f1fc1ab3ec85fe575cca9ee53a5047f4d3f7b76cea3b650b7
|
4
|
+
data.tar.gz: c29354b2de2f207185fc06fd86421b53d328aaf67d611109a6f95e9130ac5967
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 93351a3c4007145ff317257f82b764708b445e94b848d31f783f4d72071b097a8309322c4bedfdb7f321b0613ef436a314ad4e650928d0855143d3a5b98d9c13
|
7
|
+
data.tar.gz: 9119039548264867fa012bcbb90999f4ad82170c28539d43b0145687a761d508630da538ac87b75f9bf32f6b33cb0e13c22f50143248b85cc90289d5d3451d3f
|
data/CHANGELOG.md
CHANGED
data/Gemfile.lock
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
PATH
|
2
2
|
remote: .
|
3
3
|
specs:
|
4
|
-
polyphony (0.
|
4
|
+
polyphony (0.51.0)
|
5
5
|
|
6
6
|
GEM
|
7
7
|
remote: https://rubygems.org/
|
@@ -72,6 +72,7 @@ GEM
|
|
72
72
|
builder
|
73
73
|
minitest (>= 5.0)
|
74
74
|
ruby-progressbar
|
75
|
+
msgpack (1.4.2)
|
75
76
|
multi_xml (0.6.0)
|
76
77
|
mysql2 (0.5.3)
|
77
78
|
parallel (1.19.1)
|
@@ -136,6 +137,7 @@ DEPENDENCIES
|
|
136
137
|
just-the-docs (~> 0.3.0)
|
137
138
|
minitest (= 5.13.0)
|
138
139
|
minitest-reporters (= 1.4.2)
|
140
|
+
msgpack (= 1.4.2)
|
139
141
|
mysql2 (= 0.5.3)
|
140
142
|
pg (= 1.1.4)
|
141
143
|
polyphony!
|
@@ -107,6 +107,13 @@ inline VALUE backend_snooze() {
|
|
107
107
|
READ_LOOP_PREPARE_STR(); \
|
108
108
|
}
|
109
109
|
|
110
|
+
#define READ_LOOP_PASS_STR_TO_RECEIVER(receiver, method_id) { \
|
111
|
+
io_set_read_length(str, total, shrinkable); \
|
112
|
+
io_enc_str(str, fptr); \
|
113
|
+
rb_funcall_passing_block(receiver, method_id, 1, &str); \
|
114
|
+
READ_LOOP_PREPARE_STR(); \
|
115
|
+
}
|
116
|
+
|
110
117
|
inline void rectify_io_file_pos(rb_io_t *fptr) {
|
111
118
|
// Apparently after reopening a closed file, the file position is not reset,
|
112
119
|
// which causes the read to fail. Fortunately we can use fptr->rbuf.len to
|
@@ -389,6 +389,52 @@ VALUE Backend_read_loop(VALUE self, VALUE io) {
|
|
389
389
|
return io;
|
390
390
|
}
|
391
391
|
|
392
|
+
VALUE Backend_feed_loop(VALUE self, VALUE io, VALUE receiver, VALUE method) {
|
393
|
+
Backend_t *backend;
|
394
|
+
rb_io_t *fptr;
|
395
|
+
VALUE str;
|
396
|
+
long total;
|
397
|
+
long len = 8192;
|
398
|
+
int shrinkable;
|
399
|
+
char *buf;
|
400
|
+
VALUE underlying_io = rb_ivar_get(io, ID_ivar_io);
|
401
|
+
ID method_id = SYM2ID(method);
|
402
|
+
|
403
|
+
READ_LOOP_PREPARE_STR();
|
404
|
+
|
405
|
+
GetBackend(self, backend);
|
406
|
+
if (underlying_io != Qnil) io = underlying_io;
|
407
|
+
GetOpenFile(io, fptr);
|
408
|
+
rb_io_check_byte_readable(fptr);
|
409
|
+
rectify_io_file_pos(fptr);
|
410
|
+
|
411
|
+
while (1) {
|
412
|
+
VALUE resume_value = Qnil;
|
413
|
+
op_context_t *ctx = OP_CONTEXT_ACQUIRE(&backend->store, OP_READ);
|
414
|
+
struct io_uring_sqe *sqe = io_uring_get_sqe(&backend->ring);
|
415
|
+
io_uring_prep_read(sqe, fptr->fd, buf, len, -1);
|
416
|
+
|
417
|
+
ssize_t result = io_uring_backend_defer_submit_and_await(backend, sqe, ctx, &resume_value);
|
418
|
+
OP_CONTEXT_RELEASE(&backend->store, ctx);
|
419
|
+
RAISE_IF_EXCEPTION(resume_value);
|
420
|
+
if (!ctx->completed) return resume_value;
|
421
|
+
RB_GC_GUARD(resume_value);
|
422
|
+
|
423
|
+
if (result < 0)
|
424
|
+
rb_syserr_fail(-result, strerror(-result));
|
425
|
+
else if (!result)
|
426
|
+
break; // EOF
|
427
|
+
else {
|
428
|
+
total = result;
|
429
|
+
READ_LOOP_PASS_STR_TO_RECEIVER(receiver, method_id);
|
430
|
+
}
|
431
|
+
}
|
432
|
+
|
433
|
+
RB_GC_GUARD(str);
|
434
|
+
|
435
|
+
return io;
|
436
|
+
}
|
437
|
+
|
392
438
|
VALUE Backend_write(VALUE self, VALUE io, VALUE str) {
|
393
439
|
Backend_t *backend;
|
394
440
|
rb_io_t *fptr;
|
@@ -596,6 +642,51 @@ VALUE Backend_recv_loop(VALUE self, VALUE io) {
|
|
596
642
|
return io;
|
597
643
|
}
|
598
644
|
|
645
|
+
VALUE Backend_recv_feed_loop(VALUE self, VALUE io, VALUE receiver, VALUE method) {
|
646
|
+
Backend_t *backend;
|
647
|
+
rb_io_t *fptr;
|
648
|
+
VALUE str;
|
649
|
+
long total;
|
650
|
+
long len = 8192;
|
651
|
+
int shrinkable;
|
652
|
+
char *buf;
|
653
|
+
VALUE underlying_io = rb_ivar_get(io, ID_ivar_io);
|
654
|
+
ID method_id = SYM2ID(method);
|
655
|
+
|
656
|
+
READ_LOOP_PREPARE_STR();
|
657
|
+
|
658
|
+
GetBackend(self, backend);
|
659
|
+
if (underlying_io != Qnil) io = underlying_io;
|
660
|
+
GetOpenFile(io, fptr);
|
661
|
+
rb_io_check_byte_readable(fptr);
|
662
|
+
rectify_io_file_pos(fptr);
|
663
|
+
|
664
|
+
while (1) {
|
665
|
+
VALUE resume_value = Qnil;
|
666
|
+
op_context_t *ctx = OP_CONTEXT_ACQUIRE(&backend->store, OP_RECV);
|
667
|
+
struct io_uring_sqe *sqe = io_uring_get_sqe(&backend->ring);
|
668
|
+
io_uring_prep_recv(sqe, fptr->fd, buf, len, 0);
|
669
|
+
|
670
|
+
int result = io_uring_backend_defer_submit_and_await(backend, sqe, ctx, &resume_value);
|
671
|
+
OP_CONTEXT_RELEASE(&backend->store, ctx);
|
672
|
+
RAISE_IF_EXCEPTION(resume_value);
|
673
|
+
if (!ctx->completed) return resume_value;
|
674
|
+
RB_GC_GUARD(resume_value);
|
675
|
+
|
676
|
+
if (result < 0)
|
677
|
+
rb_syserr_fail(-result, strerror(-result));
|
678
|
+
else if (!result)
|
679
|
+
break; // EOF
|
680
|
+
else {
|
681
|
+
total = result;
|
682
|
+
READ_LOOP_PASS_STR_TO_RECEIVER(receiver, method_id);
|
683
|
+
}
|
684
|
+
}
|
685
|
+
|
686
|
+
RB_GC_GUARD(str);
|
687
|
+
return io;
|
688
|
+
}
|
689
|
+
|
599
690
|
VALUE Backend_send(VALUE self, VALUE io, VALUE str) {
|
600
691
|
Backend_t *backend;
|
601
692
|
rb_io_t *fptr;
|
@@ -918,9 +1009,11 @@ void Init_Backend() {
|
|
918
1009
|
|
919
1010
|
rb_define_method(cBackend, "read", Backend_read, 4);
|
920
1011
|
rb_define_method(cBackend, "read_loop", Backend_read_loop, 1);
|
1012
|
+
rb_define_method(cBackend, "feed_loop", Backend_feed_loop, 3);
|
921
1013
|
rb_define_method(cBackend, "write", Backend_write_m, -1);
|
922
1014
|
rb_define_method(cBackend, "recv", Backend_recv, 3);
|
923
1015
|
rb_define_method(cBackend, "recv_loop", Backend_recv_loop, 1);
|
1016
|
+
rb_define_method(cBackend, "recv_feed_loop", Backend_recv_feed_loop, 3);
|
924
1017
|
rb_define_method(cBackend, "send", Backend_send, 2);
|
925
1018
|
rb_define_method(cBackend, "accept", Backend_accept, 2);
|
926
1019
|
rb_define_method(cBackend, "accept_loop", Backend_accept_loop, 2);
|
@@ -334,6 +334,58 @@ error:
|
|
334
334
|
return RAISE_EXCEPTION(switchpoint_result);
|
335
335
|
}
|
336
336
|
|
337
|
+
VALUE Backend_feed_loop(VALUE self, VALUE io, VALUE receiver, VALUE method) {
|
338
|
+
Backend_t *backend;
|
339
|
+
struct libev_io watcher;
|
340
|
+
rb_io_t *fptr;
|
341
|
+
VALUE str;
|
342
|
+
long total;
|
343
|
+
long len = 8192;
|
344
|
+
int shrinkable;
|
345
|
+
char *buf;
|
346
|
+
VALUE switchpoint_result = Qnil;
|
347
|
+
VALUE underlying_io = rb_ivar_get(io, ID_ivar_io);
|
348
|
+
ID method_id = SYM2ID(method);
|
349
|
+
|
350
|
+
READ_LOOP_PREPARE_STR();
|
351
|
+
|
352
|
+
GetBackend(self, backend);
|
353
|
+
if (underlying_io != Qnil) io = underlying_io;
|
354
|
+
GetOpenFile(io, fptr);
|
355
|
+
rb_io_check_byte_readable(fptr);
|
356
|
+
io_set_nonblock(fptr, io);
|
357
|
+
rectify_io_file_pos(fptr);
|
358
|
+
watcher.fiber = Qnil;
|
359
|
+
|
360
|
+
while (1) {
|
361
|
+
ssize_t n = read(fptr->fd, buf, len);
|
362
|
+
if (n < 0) {
|
363
|
+
int e = errno;
|
364
|
+
if ((e != EWOULDBLOCK && e != EAGAIN)) rb_syserr_fail(e, strerror(e));
|
365
|
+
|
366
|
+
switchpoint_result = libev_wait_fd_with_watcher(backend, fptr->fd, &watcher, EV_READ);
|
367
|
+
if (TEST_EXCEPTION(switchpoint_result)) goto error;
|
368
|
+
}
|
369
|
+
else {
|
370
|
+
switchpoint_result = backend_snooze();
|
371
|
+
|
372
|
+
if (TEST_EXCEPTION(switchpoint_result)) goto error;
|
373
|
+
|
374
|
+
if (n == 0) break; // EOF
|
375
|
+
total = n;
|
376
|
+
READ_LOOP_PASS_STR_TO_RECEIVER(receiver, method_id);
|
377
|
+
}
|
378
|
+
}
|
379
|
+
|
380
|
+
RB_GC_GUARD(str);
|
381
|
+
RB_GC_GUARD(watcher.fiber);
|
382
|
+
RB_GC_GUARD(switchpoint_result);
|
383
|
+
|
384
|
+
return io;
|
385
|
+
error:
|
386
|
+
return RAISE_EXCEPTION(switchpoint_result);
|
387
|
+
}
|
388
|
+
|
337
389
|
VALUE Backend_write(VALUE self, VALUE io, VALUE str) {
|
338
390
|
Backend_t *backend;
|
339
391
|
struct libev_io watcher;
|
@@ -811,12 +863,14 @@ void Init_Backend() {
|
|
811
863
|
|
812
864
|
rb_define_method(cBackend, "read", Backend_read, 4);
|
813
865
|
rb_define_method(cBackend, "read_loop", Backend_read_loop, 1);
|
866
|
+
rb_define_method(cBackend, "feed_loop", Backend_feed_loop, 3);
|
814
867
|
rb_define_method(cBackend, "write", Backend_write_m, -1);
|
815
868
|
rb_define_method(cBackend, "accept", Backend_accept, 2);
|
816
869
|
rb_define_method(cBackend, "accept_loop", Backend_accept_loop, 2);
|
817
870
|
rb_define_method(cBackend, "connect", Backend_connect, 3);
|
818
871
|
rb_define_method(cBackend, "recv", Backend_recv, 3);
|
819
872
|
rb_define_method(cBackend, "recv_loop", Backend_read_loop, 1);
|
873
|
+
rb_define_method(cBackend, "recv_feed_loop", Backend_feed_loop, 3);
|
820
874
|
rb_define_method(cBackend, "send", Backend_write, 2);
|
821
875
|
rb_define_method(cBackend, "wait_io", Backend_wait_io, 2);
|
822
876
|
rb_define_method(cBackend, "sleep", Backend_sleep, 1);
|
@@ -220,6 +220,10 @@ class ::IO
|
|
220
220
|
Thread.current.backend.read_loop(self, &block)
|
221
221
|
end
|
222
222
|
|
223
|
+
def feed_loop(receiver, method, &block)
|
224
|
+
Thread.current.backend.feed_loop(self, receiver, method, &block)
|
225
|
+
end
|
226
|
+
|
223
227
|
# alias_method :orig_read, :read
|
224
228
|
# def read(length = nil, outbuf = nil)
|
225
229
|
# if length
|
@@ -31,6 +31,10 @@ class ::Socket
|
|
31
31
|
end
|
32
32
|
alias_method :read_loop, :recv_loop
|
33
33
|
|
34
|
+
def feed_loop(receiver, method, &block)
|
35
|
+
Thread.current.backend.recv_feed_loop(self, receiver, method, &block)
|
36
|
+
end
|
37
|
+
|
34
38
|
def recvfrom(maxlen, flags = 0)
|
35
39
|
@read_buffer ||= +''
|
36
40
|
while true
|
@@ -146,6 +150,10 @@ class ::TCPSocket
|
|
146
150
|
end
|
147
151
|
alias_method :read_loop, :recv_loop
|
148
152
|
|
153
|
+
def feed_loop(receiver, method, &block)
|
154
|
+
Thread.current.backend.recv_feed_loop(self, receiver, method, &block)
|
155
|
+
end
|
156
|
+
|
149
157
|
def send(mesg, flags = 0)
|
150
158
|
Thread.current.backend.send(self, mesg)
|
151
159
|
end
|
@@ -227,6 +235,10 @@ class ::UNIXSocket
|
|
227
235
|
end
|
228
236
|
alias_method :read_loop, :recv_loop
|
229
237
|
|
238
|
+
def feed_loop(receiver, method, &block)
|
239
|
+
Thread.current.backend.recv_feed_loop(self, receiver, method, &block)
|
240
|
+
end
|
241
|
+
|
230
242
|
def send(mesg, flags = 0)
|
231
243
|
Thread.current.backend.send(self, mesg)
|
232
244
|
end
|
data/lib/polyphony/version.rb
CHANGED
data/polyphony.gemspec
CHANGED
@@ -28,6 +28,7 @@ Gem::Specification.new do |s|
|
|
28
28
|
s.add_development_dependency 'rubocop', '0.85.1'
|
29
29
|
s.add_development_dependency 'pry', '0.13.1'
|
30
30
|
|
31
|
+
s.add_development_dependency 'msgpack', '1.4.2'
|
31
32
|
s.add_development_dependency 'pg', '1.1.4'
|
32
33
|
s.add_development_dependency 'redis', '4.1.0'
|
33
34
|
s.add_development_dependency 'hiredis', '0.6.3'
|
data/test/test_io.rb
CHANGED
@@ -1,6 +1,7 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
3
|
require_relative 'helper'
|
4
|
+
require 'msgpack'
|
4
5
|
|
5
6
|
class IOTest < MiniTest::Test
|
6
7
|
def setup
|
@@ -110,7 +111,7 @@ class IOTest < MiniTest::Test
|
|
110
111
|
assert_equal [], buf
|
111
112
|
|
112
113
|
o << "ulous\n"
|
113
|
-
|
114
|
+
sleep 0.01
|
114
115
|
assert_equal ["fabulous\n"], buf
|
115
116
|
|
116
117
|
o.close
|
@@ -173,6 +174,57 @@ class IOTest < MiniTest::Test
|
|
173
174
|
|
174
175
|
assert_equal 'hello: world', buf
|
175
176
|
end
|
177
|
+
|
178
|
+
def test_feed_loop_with_block
|
179
|
+
i, o = IO.pipe
|
180
|
+
unpacker = MessagePack::Unpacker.new
|
181
|
+
buffer = []
|
182
|
+
reader = spin do
|
183
|
+
i.feed_loop(unpacker, :feed_each) { |msg| buffer << msg }
|
184
|
+
end
|
185
|
+
o << 'foo'.to_msgpack
|
186
|
+
sleep 0.01
|
187
|
+
assert_equal ['foo'], buffer
|
188
|
+
|
189
|
+
o << 'bar'.to_msgpack
|
190
|
+
sleep 0.01
|
191
|
+
assert_equal ['foo', 'bar'], buffer
|
192
|
+
|
193
|
+
o << 'baz'.to_msgpack
|
194
|
+
sleep 0.01
|
195
|
+
assert_equal ['foo', 'bar', 'baz'], buffer
|
196
|
+
end
|
197
|
+
|
198
|
+
class Receiver
|
199
|
+
attr_reader :buffer
|
200
|
+
|
201
|
+
def initialize
|
202
|
+
@buffer = []
|
203
|
+
end
|
204
|
+
|
205
|
+
def recv(obj)
|
206
|
+
@buffer << obj
|
207
|
+
end
|
208
|
+
end
|
209
|
+
|
210
|
+
def test_feed_loop_without_block
|
211
|
+
i, o = IO.pipe
|
212
|
+
receiver = Receiver.new
|
213
|
+
reader = spin do
|
214
|
+
i.feed_loop(receiver, :recv)
|
215
|
+
end
|
216
|
+
o << 'foo'
|
217
|
+
sleep 0.01
|
218
|
+
assert_equal ['foo'], receiver.buffer
|
219
|
+
|
220
|
+
o << 'bar'
|
221
|
+
sleep 0.01
|
222
|
+
assert_equal ['foo', 'bar'], receiver.buffer
|
223
|
+
|
224
|
+
o << 'baz'
|
225
|
+
sleep 0.01
|
226
|
+
assert_equal ['foo', 'bar', 'baz'], receiver.buffer
|
227
|
+
end
|
176
228
|
end
|
177
229
|
|
178
230
|
class IOClassMethodsTest < MiniTest::Test
|
data/test/test_socket.rb
CHANGED
@@ -2,6 +2,7 @@
|
|
2
2
|
|
3
3
|
require_relative 'helper'
|
4
4
|
require 'fileutils'
|
5
|
+
require 'msgpack'
|
5
6
|
|
6
7
|
class SocketTest < MiniTest::Test
|
7
8
|
def setup
|
@@ -33,6 +34,39 @@ class SocketTest < MiniTest::Test
|
|
33
34
|
server&.close
|
34
35
|
end
|
35
36
|
|
37
|
+
def test_feed_loop
|
38
|
+
port = rand(1234..5678)
|
39
|
+
server = TCPServer.new('127.0.0.1', port)
|
40
|
+
|
41
|
+
server_fiber = spin do
|
42
|
+
reader = MessagePack::Unpacker.new
|
43
|
+
while (socket = server.accept)
|
44
|
+
spin do
|
45
|
+
socket.feed_loop(reader, :feed_each) do |msg|
|
46
|
+
msg = { 'result' => msg['x'] + msg['y'] }
|
47
|
+
socket << msg.to_msgpack
|
48
|
+
end
|
49
|
+
end
|
50
|
+
end
|
51
|
+
end
|
52
|
+
|
53
|
+
snooze
|
54
|
+
client = TCPSocket.new('127.0.0.1', port)
|
55
|
+
reader = MessagePack::Unpacker.new
|
56
|
+
client << { 'x' => 13, 'y' => 14 }.to_msgpack
|
57
|
+
result = nil
|
58
|
+
client.feed_loop(reader, :feed_each) do |msg|
|
59
|
+
result = msg
|
60
|
+
break
|
61
|
+
end
|
62
|
+
assert_equal({ 'result' => 27}, result)
|
63
|
+
client.close
|
64
|
+
ensure
|
65
|
+
server_fiber&.stop
|
66
|
+
server_fiber&.await
|
67
|
+
server&.close
|
68
|
+
end
|
69
|
+
|
36
70
|
def test_unix_socket
|
37
71
|
path = '/tmp/test_unix_socket'
|
38
72
|
FileUtils.rm(path) rescue nil
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: polyphony
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 0.51.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Sharon Rosner
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2021-
|
11
|
+
date: 2021-02-02 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: rake-compiler
|
@@ -94,6 +94,20 @@ dependencies:
|
|
94
94
|
- - '='
|
95
95
|
- !ruby/object:Gem::Version
|
96
96
|
version: 0.13.1
|
97
|
+
- !ruby/object:Gem::Dependency
|
98
|
+
name: msgpack
|
99
|
+
requirement: !ruby/object:Gem::Requirement
|
100
|
+
requirements:
|
101
|
+
- - '='
|
102
|
+
- !ruby/object:Gem::Version
|
103
|
+
version: 1.4.2
|
104
|
+
type: :development
|
105
|
+
prerelease: false
|
106
|
+
version_requirements: !ruby/object:Gem::Requirement
|
107
|
+
requirements:
|
108
|
+
- - '='
|
109
|
+
- !ruby/object:Gem::Version
|
110
|
+
version: 1.4.2
|
97
111
|
- !ruby/object:Gem::Dependency
|
98
112
|
name: pg
|
99
113
|
requirement: !ruby/object:Gem::Requirement
|