polyphony 0.98 → 0.99.1
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.github/FUNDING.yml +1 -1
- data/.rubocop.yml +3 -3
- data/.yardopts +30 -0
- data/CHANGELOG.md +11 -0
- data/LICENSE +1 -1
- data/README.md +63 -29
- data/Rakefile +1 -5
- data/TODO.md +0 -4
- data/docs/{main-concepts/concurrency.md → concurrency.md} +2 -9
- data/docs/{main-concepts/design-principles.md → design-principles.md} +3 -9
- data/docs/{main-concepts/exception-handling.md → exception-handling.md} +2 -9
- data/docs/{main-concepts/extending.md → extending.md} +2 -9
- data/docs/faq.md +3 -16
- data/docs/{main-concepts/fiber-scheduling.md → fiber-scheduling.md} +1 -9
- data/docs/link_rewriter.rb +16 -0
- data/docs/{getting-started/overview.md → overview.md} +1 -30
- data/docs/{getting-started/tutorial.md → tutorial.md} +3 -28
- data/docs/{_posts/2020-07-26-polyphony-0.44.md → whats-new.md} +3 -1
- data/examples/adapters/redis_client.rb +3 -2
- data/examples/io/echo_server.rb +1 -1
- data/examples/io/echo_server_plain_ruby.rb +26 -0
- data/examples/io/https_server_sni_2.rb +14 -8
- data/ext/polyphony/backend_io_uring.c +154 -9
- data/ext/polyphony/backend_io_uring_context.c +21 -12
- data/ext/polyphony/backend_io_uring_context.h +12 -7
- data/ext/polyphony/backend_libev.c +1 -1
- data/ext/polyphony/extconf.rb +25 -8
- data/ext/polyphony/fiber.c +79 -2
- data/ext/polyphony/io_extensions.c +53 -0
- data/ext/polyphony/libev.h +0 -2
- data/ext/polyphony/pipe.c +42 -2
- data/ext/polyphony/polyphony.c +345 -31
- data/ext/polyphony/polyphony.h +9 -2
- data/ext/polyphony/queue.c +181 -0
- data/ext/polyphony/ring_buffer.c +0 -1
- data/ext/polyphony/runqueue.c +8 -1
- data/ext/polyphony/runqueue_ring_buffer.c +13 -0
- data/ext/polyphony/runqueue_ring_buffer.h +2 -1
- data/ext/polyphony/socket_extensions.c +6 -0
- data/ext/polyphony/thread.c +34 -2
- data/lib/polyphony/adapters/process.rb +11 -1
- data/lib/polyphony/adapters/sequel.rb +1 -1
- data/lib/polyphony/core/channel.rb +2 -0
- data/lib/polyphony/core/debug.rb +1 -1
- data/lib/polyphony/core/global_api.rb +25 -24
- data/lib/polyphony/core/resource_pool.rb +7 -6
- data/lib/polyphony/core/sync.rb +55 -2
- data/lib/polyphony/core/thread_pool.rb +3 -3
- data/lib/polyphony/core/timer.rb +8 -8
- data/lib/polyphony/extensions/exception.rb +2 -0
- data/lib/polyphony/extensions/fiber.rb +15 -13
- data/lib/polyphony/extensions/io.rb +161 -16
- data/lib/polyphony/extensions/kernel.rb +20 -2
- data/lib/polyphony/extensions/openssl.rb +101 -12
- data/lib/polyphony/extensions/pipe.rb +103 -7
- data/lib/polyphony/extensions/process.rb +13 -1
- data/lib/polyphony/extensions/socket.rb +93 -27
- data/lib/polyphony/extensions/thread.rb +9 -1
- data/lib/polyphony/extensions/timeout.rb +1 -1
- data/lib/polyphony/version.rb +2 -1
- data/lib/polyphony.rb +27 -7
- data/polyphony.gemspec +1 -8
- data/test/stress.rb +1 -1
- data/test/test_global_api.rb +45 -7
- data/test/test_io.rb +6 -7
- data/test/test_socket.rb +157 -0
- data/test/test_sync.rb +42 -1
- data/test/test_timer.rb +5 -5
- data/vendor/liburing/.github/workflows/build.yml +7 -16
- data/vendor/liburing/.gitignore +5 -0
- data/vendor/liburing/CHANGELOG +23 -1
- data/vendor/liburing/Makefile +4 -3
- data/vendor/liburing/Makefile.common +1 -0
- data/vendor/liburing/README +48 -0
- data/vendor/liburing/configure +76 -6
- data/vendor/liburing/debian/changelog +11 -0
- data/vendor/liburing/debian/control +7 -16
- data/vendor/liburing/debian/liburing-dev.manpages +3 -6
- data/vendor/liburing/debian/liburing2.install +1 -0
- data/vendor/liburing/debian/liburing2.symbols +56 -0
- data/vendor/liburing/debian/rules +15 -68
- data/vendor/liburing/examples/Makefile +4 -0
- data/vendor/liburing/examples/io_uring-close-test.c +123 -0
- data/vendor/liburing/examples/io_uring-udp.c +1 -1
- data/vendor/liburing/examples/send-zerocopy.c +315 -56
- data/vendor/liburing/examples/ucontext-cp.c +2 -17
- data/vendor/liburing/liburing-ffi.pc.in +12 -0
- data/vendor/liburing/liburing.pc.in +1 -1
- data/vendor/liburing/liburing.spec +1 -1
- data/vendor/liburing/make-debs.sh +3 -3
- data/vendor/liburing/man/IO_URING_CHECK_VERSION.3 +1 -0
- data/vendor/liburing/man/IO_URING_VERSION_MAJOR.3 +1 -0
- data/vendor/liburing/man/IO_URING_VERSION_MINOR.3 +1 -0
- data/vendor/liburing/man/io_uring_buf_ring_add.3 +6 -6
- data/vendor/liburing/man/io_uring_check_version.3 +72 -0
- data/vendor/liburing/man/io_uring_close_ring_fd.3 +43 -0
- data/vendor/liburing/man/io_uring_major_version.3 +1 -0
- data/vendor/liburing/man/io_uring_minor_version.3 +1 -0
- data/vendor/liburing/man/io_uring_prep_accept.3 +1 -1
- data/vendor/liburing/man/io_uring_prep_fgetxattr.3 +1 -0
- data/vendor/liburing/man/io_uring_prep_fsetxattr.3 +1 -0
- data/vendor/liburing/man/io_uring_prep_getxattr.3 +61 -0
- data/vendor/liburing/man/io_uring_prep_link_timeout.3 +94 -0
- data/vendor/liburing/man/io_uring_prep_msg_ring.3 +22 -2
- data/vendor/liburing/man/io_uring_prep_msg_ring_cqe_flags.3 +1 -0
- data/vendor/liburing/man/io_uring_prep_poll_add.3 +1 -1
- data/vendor/liburing/man/io_uring_prep_provide_buffers.3 +18 -9
- data/vendor/liburing/man/io_uring_prep_readv.3 +3 -3
- data/vendor/liburing/man/io_uring_prep_readv2.3 +3 -3
- data/vendor/liburing/man/io_uring_prep_recv.3 +5 -5
- data/vendor/liburing/man/io_uring_prep_recvmsg.3 +4 -4
- data/vendor/liburing/man/io_uring_prep_send.3 +9 -0
- data/vendor/liburing/man/io_uring_prep_send_set_addr.3 +38 -0
- data/vendor/liburing/man/io_uring_prep_send_zc.3 +39 -7
- data/vendor/liburing/man/io_uring_prep_send_zc_fixed.3 +1 -0
- data/vendor/liburing/man/io_uring_prep_sendmsg.3 +20 -0
- data/vendor/liburing/man/io_uring_prep_sendmsg_zc.3 +1 -0
- data/vendor/liburing/man/io_uring_prep_setxattr.3 +64 -0
- data/vendor/liburing/man/io_uring_prep_splice.3 +40 -0
- data/vendor/liburing/man/io_uring_prep_writev.3 +2 -2
- data/vendor/liburing/man/io_uring_prep_writev2.3 +2 -2
- data/vendor/liburing/man/io_uring_recvmsg_out.3 +13 -9
- data/vendor/liburing/man/io_uring_register.2 +15 -9
- data/vendor/liburing/man/io_uring_register_buf_ring.3 +4 -4
- data/vendor/liburing/man/io_uring_register_buffers.3 +49 -6
- data/vendor/liburing/man/io_uring_register_buffers_sparse.3 +1 -0
- data/vendor/liburing/man/io_uring_register_buffers_tags.3 +1 -0
- data/vendor/liburing/man/io_uring_register_buffers_update_tag.3 +1 -0
- data/vendor/liburing/man/io_uring_register_files.3 +60 -5
- data/vendor/liburing/man/io_uring_register_files_tags.3 +1 -0
- data/vendor/liburing/man/io_uring_register_files_update.3 +1 -0
- data/vendor/liburing/man/io_uring_register_files_update_tag.3 +1 -0
- data/vendor/liburing/man/io_uring_setup.2 +31 -2
- data/vendor/liburing/man/io_uring_wait_cqe_timeout.3 +1 -1
- data/vendor/liburing/src/Makefile +25 -3
- data/vendor/liburing/src/ffi.c +15 -0
- data/vendor/liburing/src/include/liburing/io_uring.h +30 -7
- data/vendor/liburing/src/include/liburing.h +190 -148
- data/vendor/liburing/src/int_flags.h +1 -0
- data/vendor/liburing/src/lib.h +5 -16
- data/vendor/liburing/src/liburing-ffi.map +172 -0
- data/vendor/liburing/src/liburing.map +11 -0
- data/vendor/liburing/src/nolibc.c +9 -2
- data/vendor/liburing/src/queue.c +2 -2
- data/vendor/liburing/src/register.c +66 -96
- data/vendor/liburing/src/setup.c +5 -4
- data/vendor/liburing/src/version.c +21 -0
- data/vendor/liburing/test/232c93d07b74.c +3 -3
- data/vendor/liburing/test/35fa71a030ca.c +3 -3
- data/vendor/liburing/test/500f9fbadef8.c +2 -0
- data/vendor/liburing/test/917257daa0fe.c +1 -1
- data/vendor/liburing/test/Makefile +27 -7
- data/vendor/liburing/test/a0908ae19763.c +2 -2
- data/vendor/liburing/test/a4c0b3decb33.c +2 -2
- data/vendor/liburing/test/accept-link.c +4 -4
- data/vendor/liburing/test/accept-reuse.c +5 -7
- data/vendor/liburing/test/accept.c +34 -31
- data/vendor/liburing/test/b19062a56726.c +1 -1
- data/vendor/liburing/test/buf-ring.c +58 -4
- data/vendor/liburing/test/ce593a6c480a.c +2 -2
- data/vendor/liburing/test/close-opath.c +2 -1
- data/vendor/liburing/test/connect.c +8 -0
- data/vendor/liburing/test/cq-overflow.c +14 -8
- data/vendor/liburing/test/d4ae271dfaae.c +1 -1
- data/vendor/liburing/test/defer-taskrun.c +64 -9
- data/vendor/liburing/test/defer.c +1 -1
- data/vendor/liburing/test/double-poll-crash.c +3 -3
- data/vendor/liburing/test/eeed8b54e0df.c +8 -3
- data/vendor/liburing/test/eploop.c +74 -0
- data/vendor/liburing/test/eventfd-ring.c +1 -1
- data/vendor/liburing/test/eventfd.c +1 -1
- data/vendor/liburing/test/evloop.c +73 -0
- data/vendor/liburing/test/exit-no-cleanup.c +1 -1
- data/vendor/liburing/test/fadvise.c +1 -1
- data/vendor/liburing/test/fc2a85cb02ef.c +3 -3
- data/vendor/liburing/test/fd-pass.c +35 -16
- data/vendor/liburing/test/file-register.c +61 -0
- data/vendor/liburing/test/file-verify.c +2 -2
- data/vendor/liburing/test/files-exit-hang-timeout.c +2 -2
- data/vendor/liburing/test/fixed-link.c +1 -1
- data/vendor/liburing/test/fsnotify.c +118 -0
- data/vendor/liburing/test/hardlink.c +1 -1
- data/vendor/liburing/test/helpers.c +54 -2
- data/vendor/liburing/test/helpers.h +4 -0
- data/vendor/liburing/test/io-cancel.c +3 -1
- data/vendor/liburing/test/io_uring_passthrough.c +39 -8
- data/vendor/liburing/test/io_uring_setup.c +3 -80
- data/vendor/liburing/test/iopoll-overflow.c +118 -0
- data/vendor/liburing/test/iopoll.c +90 -4
- data/vendor/liburing/test/lfs-openat-write.c +7 -9
- data/vendor/liburing/test/lfs-openat.c +6 -8
- data/vendor/liburing/test/link_drain.c +31 -5
- data/vendor/liburing/test/madvise.c +1 -1
- data/vendor/liburing/test/msg-ring-flags.c +192 -0
- data/vendor/liburing/test/msg-ring-overflow.c +159 -0
- data/vendor/liburing/test/msg-ring.c +173 -13
- data/vendor/liburing/test/multicqes_drain.c +22 -19
- data/vendor/liburing/test/nvme.h +4 -3
- data/vendor/liburing/test/pipe-bug.c +95 -0
- data/vendor/liburing/test/poll-link.c +3 -3
- data/vendor/liburing/test/poll-many.c +41 -19
- data/vendor/liburing/test/poll-mshot-overflow.c +105 -2
- data/vendor/liburing/test/poll-race-mshot.c +292 -0
- data/vendor/liburing/test/poll-race.c +105 -0
- data/vendor/liburing/test/poll.c +244 -26
- data/vendor/liburing/test/pollfree.c +5 -5
- data/vendor/liburing/test/read-before-exit.c +20 -3
- data/vendor/liburing/test/read-write.c +2 -0
- data/vendor/liburing/test/recv-multishot.c +96 -3
- data/vendor/liburing/test/reg-reg-ring.c +90 -0
- data/vendor/liburing/test/rename.c +1 -1
- data/vendor/liburing/test/ring-leak.c +0 -1
- data/vendor/liburing/test/ring-leak2.c +1 -1
- data/vendor/liburing/test/ringbuf-read.c +10 -6
- data/vendor/liburing/test/send-zerocopy.c +273 -103
- data/vendor/liburing/test/send_recv.c +7 -4
- data/vendor/liburing/test/sendmsg_fs_cve.c +2 -2
- data/vendor/liburing/test/single-issuer.c +7 -9
- data/vendor/liburing/test/skip-cqe.c +3 -4
- data/vendor/liburing/test/socket.c +0 -1
- data/vendor/liburing/test/sq-poll-dup.c +10 -3
- data/vendor/liburing/test/sq-poll-kthread.c +1 -1
- data/vendor/liburing/test/sq-poll-share.c +3 -2
- data/vendor/liburing/test/sqpoll-cancel-hang.c +17 -6
- data/vendor/liburing/test/sqpoll-disable-exit.c +4 -4
- data/vendor/liburing/test/symlink.c +2 -1
- data/vendor/liburing/test/test.h +2 -1
- data/vendor/liburing/test/timeout-new.c +11 -7
- data/vendor/liburing/test/timeout.c +1 -2
- data/vendor/liburing/test/unlink.c +1 -1
- data/vendor/liburing/test/version.c +25 -0
- data/vendor/liburing/test/wakeup-hang.c +1 -1
- data/vendor/liburing/test/xattr.c +8 -4
- metadata +57 -44
- data/docs/_config.yml +0 -64
- data/docs/_includes/head.html +0 -40
- data/docs/_includes/title.html +0 -1
- data/docs/_sass/custom/custom.scss +0 -10
- data/docs/_sass/overrides.scss +0 -0
- data/docs/api-reference/exception.md +0 -31
- data/docs/api-reference/fiber.md +0 -425
- data/docs/api-reference/index.md +0 -9
- data/docs/api-reference/io.md +0 -36
- data/docs/api-reference/object.md +0 -99
- data/docs/api-reference/polyphony-baseexception.md +0 -33
- data/docs/api-reference/polyphony-cancel.md +0 -26
- data/docs/api-reference/polyphony-moveon.md +0 -24
- data/docs/api-reference/polyphony-net.md +0 -20
- data/docs/api-reference/polyphony-process.md +0 -28
- data/docs/api-reference/polyphony-resourcepool.md +0 -59
- data/docs/api-reference/polyphony-restart.md +0 -18
- data/docs/api-reference/polyphony-terminate.md +0 -18
- data/docs/api-reference/polyphony-threadpool.md +0 -67
- data/docs/api-reference/polyphony-throttler.md +0 -77
- data/docs/api-reference/polyphony.md +0 -36
- data/docs/api-reference/thread.md +0 -88
- data/docs/favicon.ico +0 -0
- data/docs/getting-started/index.md +0 -10
- data/docs/getting-started/installing.md +0 -34
- data/vendor/liburing/debian/compat +0 -1
- data/vendor/liburing/debian/liburing1-udeb.install +0 -1
- data/vendor/liburing/debian/liburing1.install +0 -1
- data/vendor/liburing/debian/liburing1.symbols +0 -32
- /data/{docs/assets/img → assets}/echo-fibers.svg +0 -0
- /data/{docs → assets}/polyphony-logo.png +0 -0
- /data/{docs/assets/img → assets}/sleeping-fiber.svg +0 -0
@@ -5,15 +5,17 @@
|
|
5
5
|
#include <stdint.h>
|
6
6
|
#include <assert.h>
|
7
7
|
#include <errno.h>
|
8
|
-
#include <error.h>
|
9
8
|
#include <limits.h>
|
10
9
|
#include <fcntl.h>
|
11
10
|
#include <unistd.h>
|
12
11
|
#include <stdbool.h>
|
12
|
+
#include <stdarg.h>
|
13
13
|
#include <string.h>
|
14
|
+
#include <pthread.h>
|
14
15
|
|
16
|
+
#include <poll.h>
|
17
|
+
#include <sched.h>
|
15
18
|
#include <arpa/inet.h>
|
16
|
-
#include <linux/errqueue.h>
|
17
19
|
#include <linux/if_packet.h>
|
18
20
|
#include <linux/ipv6.h>
|
19
21
|
#include <linux/socket.h>
|
@@ -35,27 +37,92 @@
|
|
35
37
|
#include <sys/time.h>
|
36
38
|
#include <sys/types.h>
|
37
39
|
#include <sys/wait.h>
|
40
|
+
#include <sys/mman.h>
|
41
|
+
#include <linux/mman.h>
|
38
42
|
|
39
43
|
#include "liburing.h"
|
40
44
|
|
41
45
|
#define ZC_TAG 0xfffffffULL
|
42
46
|
#define MAX_SUBMIT_NR 512
|
47
|
+
#define MAX_THREADS 100
|
48
|
+
|
49
|
+
struct thread_data {
|
50
|
+
pthread_t thread;
|
51
|
+
void *ret;
|
52
|
+
int idx;
|
53
|
+
unsigned long long packets;
|
54
|
+
unsigned long long bytes;
|
55
|
+
struct sockaddr_storage dst_addr;
|
56
|
+
int fd;
|
57
|
+
};
|
43
58
|
|
44
59
|
static bool cfg_reg_ringfd = true;
|
45
60
|
static bool cfg_fixed_files = 1;
|
46
61
|
static bool cfg_zc = 1;
|
47
62
|
static int cfg_nr_reqs = 8;
|
48
63
|
static bool cfg_fixed_buf = 1;
|
64
|
+
static bool cfg_hugetlb = 0;
|
65
|
+
static bool cfg_defer_taskrun = 0;
|
66
|
+
static int cfg_cpu = -1;
|
67
|
+
static bool cfg_rx = 0;
|
68
|
+
static unsigned cfg_nr_threads = 1;
|
49
69
|
|
50
70
|
static int cfg_family = PF_UNSPEC;
|
71
|
+
static int cfg_type = 0;
|
51
72
|
static int cfg_payload_len;
|
52
73
|
static int cfg_port = 8000;
|
53
74
|
static int cfg_runtime_ms = 4200;
|
54
75
|
|
55
76
|
static socklen_t cfg_alen;
|
56
|
-
static
|
77
|
+
static char *str_addr = NULL;
|
78
|
+
|
79
|
+
static char payload_buf[IP_MAXPACKET] __attribute__((aligned(4096)));
|
80
|
+
static char *payload;
|
81
|
+
static struct thread_data threads[MAX_THREADS];
|
82
|
+
static pthread_barrier_t barrier;
|
83
|
+
|
84
|
+
/*
|
85
|
+
* Implementation of error(3), prints an error message and exits.
|
86
|
+
*/
|
87
|
+
static void t_error(int status, int errnum, const char *format, ...)
|
88
|
+
{
|
89
|
+
va_list args;
|
90
|
+
va_start(args, format);
|
91
|
+
|
92
|
+
vfprintf(stderr, format, args);
|
93
|
+
if (errnum)
|
94
|
+
fprintf(stderr, ": %s", strerror(errnum));
|
95
|
+
|
96
|
+
fprintf(stderr, "\n");
|
97
|
+
va_end(args);
|
98
|
+
exit(status);
|
99
|
+
}
|
100
|
+
|
101
|
+
static void set_cpu_affinity(void)
|
102
|
+
{
|
103
|
+
cpu_set_t mask;
|
104
|
+
|
105
|
+
if (cfg_cpu == -1)
|
106
|
+
return;
|
107
|
+
|
108
|
+
CPU_ZERO(&mask);
|
109
|
+
CPU_SET(cfg_cpu, &mask);
|
110
|
+
if (sched_setaffinity(0, sizeof(mask), &mask))
|
111
|
+
t_error(1, errno, "unable to pin cpu\n");
|
112
|
+
}
|
113
|
+
|
114
|
+
static void set_iowq_affinity(struct io_uring *ring)
|
115
|
+
{
|
116
|
+
cpu_set_t mask;
|
117
|
+
int ret;
|
57
118
|
|
58
|
-
|
119
|
+
if (cfg_cpu == -1)
|
120
|
+
return;
|
121
|
+
|
122
|
+
ret = io_uring_register_iowq_aff(ring, 1, &mask);
|
123
|
+
if (ret)
|
124
|
+
t_error(1, ret, "unabled to set io-wq affinity\n");
|
125
|
+
}
|
59
126
|
|
60
127
|
static unsigned long gettimeofday_ms(void)
|
61
128
|
{
|
@@ -68,7 +135,7 @@ static unsigned long gettimeofday_ms(void)
|
|
68
135
|
static void do_setsockopt(int fd, int level, int optname, int val)
|
69
136
|
{
|
70
137
|
if (setsockopt(fd, level, optname, &val, sizeof(val)))
|
71
|
-
|
138
|
+
t_error(1, errno, "setsockopt %d.%d: %d", level, optname, val);
|
72
139
|
}
|
73
140
|
|
74
141
|
static void setup_sockaddr(int domain, const char *str_addr,
|
@@ -76,42 +143,156 @@ static void setup_sockaddr(int domain, const char *str_addr,
|
|
76
143
|
{
|
77
144
|
struct sockaddr_in6 *addr6 = (void *) sockaddr;
|
78
145
|
struct sockaddr_in *addr4 = (void *) sockaddr;
|
146
|
+
int port = cfg_port;
|
79
147
|
|
80
148
|
switch (domain) {
|
81
149
|
case PF_INET:
|
82
150
|
memset(addr4, 0, sizeof(*addr4));
|
83
151
|
addr4->sin_family = AF_INET;
|
84
|
-
addr4->sin_port = htons(
|
152
|
+
addr4->sin_port = htons(port);
|
85
153
|
if (str_addr &&
|
86
154
|
inet_pton(AF_INET, str_addr, &(addr4->sin_addr)) != 1)
|
87
|
-
|
155
|
+
t_error(1, 0, "ipv4 parse error: %s", str_addr);
|
88
156
|
break;
|
89
157
|
case PF_INET6:
|
90
158
|
memset(addr6, 0, sizeof(*addr6));
|
91
159
|
addr6->sin6_family = AF_INET6;
|
92
|
-
addr6->sin6_port = htons(
|
160
|
+
addr6->sin6_port = htons(port);
|
93
161
|
if (str_addr &&
|
94
162
|
inet_pton(AF_INET6, str_addr, &(addr6->sin6_addr)) != 1)
|
95
|
-
|
163
|
+
t_error(1, 0, "ipv6 parse error: %s", str_addr);
|
96
164
|
break;
|
97
165
|
default:
|
98
|
-
|
166
|
+
t_error(1, 0, "illegal domain");
|
99
167
|
}
|
100
168
|
}
|
101
169
|
|
102
|
-
static int
|
170
|
+
static int do_poll(int fd, int events)
|
103
171
|
{
|
104
|
-
|
172
|
+
struct pollfd pfd;
|
173
|
+
int ret;
|
174
|
+
|
175
|
+
pfd.events = events;
|
176
|
+
pfd.revents = 0;
|
177
|
+
pfd.fd = fd;
|
178
|
+
|
179
|
+
ret = poll(&pfd, 1, -1);
|
180
|
+
if (ret == -1)
|
181
|
+
t_error(1, errno, "poll");
|
182
|
+
|
183
|
+
return ret && (pfd.revents & events);
|
184
|
+
}
|
185
|
+
|
186
|
+
/* Flush all outstanding bytes for the tcp receive queue */
|
187
|
+
static int do_flush_tcp(struct thread_data *td, int fd)
|
188
|
+
{
|
189
|
+
int ret;
|
190
|
+
|
191
|
+
/* MSG_TRUNC flushes up to len bytes */
|
192
|
+
ret = recv(fd, NULL, 1 << 21, MSG_TRUNC | MSG_DONTWAIT);
|
193
|
+
if (ret == -1 && errno == EAGAIN)
|
194
|
+
return 0;
|
195
|
+
if (ret == -1)
|
196
|
+
t_error(1, errno, "flush");
|
197
|
+
if (!ret)
|
198
|
+
return 1;
|
199
|
+
|
200
|
+
td->packets++;
|
201
|
+
td->bytes += ret;
|
202
|
+
return 0;
|
203
|
+
}
|
204
|
+
|
205
|
+
/* Flush all outstanding datagrams. Verify first few bytes of each. */
|
206
|
+
static int do_flush_datagram(struct thread_data *td, int fd)
|
207
|
+
{
|
208
|
+
long ret, off = 0;
|
209
|
+
char buf[64];
|
210
|
+
|
211
|
+
/* MSG_TRUNC will return full datagram length */
|
212
|
+
ret = recv(fd, buf, sizeof(buf), MSG_DONTWAIT | MSG_TRUNC);
|
213
|
+
if (ret == -1 && errno == EAGAIN)
|
214
|
+
return 0;
|
215
|
+
|
216
|
+
if (ret == -1)
|
217
|
+
t_error(1, errno, "recv");
|
218
|
+
if (ret != cfg_payload_len)
|
219
|
+
t_error(1, 0, "recv: ret=%u != %u", ret, cfg_payload_len);
|
220
|
+
if ((unsigned long) ret > sizeof(buf) - off)
|
221
|
+
ret = sizeof(buf) - off;
|
222
|
+
if (memcmp(buf + off, payload, ret))
|
223
|
+
t_error(1, 0, "recv: data mismatch");
|
224
|
+
|
225
|
+
td->packets++;
|
226
|
+
td->bytes += cfg_payload_len;
|
227
|
+
return 0;
|
228
|
+
}
|
229
|
+
|
230
|
+
static void do_setup_rx(int domain, int type, int protocol)
|
231
|
+
{
|
232
|
+
struct sockaddr_storage addr = {};
|
233
|
+
struct thread_data *td;
|
234
|
+
int listen_fd, fd;
|
235
|
+
unsigned int i;
|
105
236
|
|
106
237
|
fd = socket(domain, type, protocol);
|
107
238
|
if (fd == -1)
|
108
|
-
|
239
|
+
t_error(1, errno, "socket r");
|
240
|
+
|
241
|
+
do_setsockopt(fd, SOL_SOCKET, SO_REUSEPORT, 1);
|
242
|
+
|
243
|
+
setup_sockaddr(cfg_family, str_addr, &addr);
|
109
244
|
|
110
|
-
|
245
|
+
if (bind(fd, (void *)&addr, cfg_alen))
|
246
|
+
t_error(1, errno, "bind");
|
111
247
|
|
112
|
-
if (
|
113
|
-
|
114
|
-
|
248
|
+
if (type != SOCK_STREAM) {
|
249
|
+
if (cfg_nr_threads != 1)
|
250
|
+
t_error(1, 0, "udp rx cant multithread");
|
251
|
+
threads[0].fd = fd;
|
252
|
+
return;
|
253
|
+
}
|
254
|
+
|
255
|
+
listen_fd = fd;
|
256
|
+
if (listen(listen_fd, cfg_nr_threads))
|
257
|
+
t_error(1, errno, "listen");
|
258
|
+
|
259
|
+
for (i = 0; i < cfg_nr_threads; i++) {
|
260
|
+
td = &threads[i];
|
261
|
+
|
262
|
+
fd = accept(listen_fd, NULL, NULL);
|
263
|
+
if (fd == -1)
|
264
|
+
t_error(1, errno, "accept");
|
265
|
+
td->fd = fd;
|
266
|
+
}
|
267
|
+
|
268
|
+
if (close(listen_fd))
|
269
|
+
t_error(1, errno, "close listen sock");
|
270
|
+
}
|
271
|
+
|
272
|
+
static void *do_rx(void *arg)
|
273
|
+
{
|
274
|
+
struct thread_data *td = arg;
|
275
|
+
const int cfg_receiver_wait_ms = 400;
|
276
|
+
uint64_t tstop;
|
277
|
+
int ret, fd = td->fd;
|
278
|
+
|
279
|
+
tstop = gettimeofday_ms() + cfg_runtime_ms + cfg_receiver_wait_ms;
|
280
|
+
do {
|
281
|
+
if (cfg_type == SOCK_STREAM)
|
282
|
+
ret = do_flush_tcp(td, fd);
|
283
|
+
else
|
284
|
+
ret = do_flush_datagram(td, fd);
|
285
|
+
|
286
|
+
if (ret)
|
287
|
+
break;
|
288
|
+
|
289
|
+
do_poll(fd, POLLIN);
|
290
|
+
} while (gettimeofday_ms() < tstop);
|
291
|
+
|
292
|
+
if (close(fd))
|
293
|
+
t_error(1, errno, "close");
|
294
|
+
pthread_exit(&td->ret);
|
295
|
+
return NULL;
|
115
296
|
}
|
116
297
|
|
117
298
|
static inline struct io_uring_cqe *wait_cqe_fast(struct io_uring *ring)
|
@@ -125,35 +306,46 @@ static inline struct io_uring_cqe *wait_cqe_fast(struct io_uring *ring)
|
|
125
306
|
|
126
307
|
ret = io_uring_wait_cqe(ring, &cqe);
|
127
308
|
if (ret)
|
128
|
-
|
309
|
+
t_error(1, ret, "wait cqe");
|
129
310
|
return cqe;
|
130
311
|
}
|
131
312
|
|
132
|
-
static void do_tx(int domain, int type, int protocol)
|
313
|
+
static void do_tx(struct thread_data *td, int domain, int type, int protocol)
|
133
314
|
{
|
134
|
-
|
135
|
-
unsigned long bytes = 0;
|
315
|
+
const int notif_slack = 128;
|
136
316
|
struct io_uring ring;
|
137
317
|
struct iovec iov;
|
138
318
|
uint64_t tstop;
|
139
319
|
int i, fd, ret;
|
140
320
|
int compl_cqes = 0;
|
321
|
+
int ring_flags = IORING_SETUP_COOP_TASKRUN | IORING_SETUP_SINGLE_ISSUER;
|
322
|
+
|
323
|
+
if (cfg_defer_taskrun)
|
324
|
+
ring_flags |= IORING_SETUP_DEFER_TASKRUN;
|
325
|
+
|
326
|
+
fd = socket(domain, type, protocol);
|
327
|
+
if (fd == -1)
|
328
|
+
t_error(1, errno, "socket t");
|
141
329
|
|
142
|
-
|
330
|
+
if (connect(fd, (void *)&td->dst_addr, cfg_alen))
|
331
|
+
t_error(1, errno, "connect, idx %i", td->idx);
|
143
332
|
|
144
|
-
ret = io_uring_queue_init(512, &ring,
|
333
|
+
ret = io_uring_queue_init(512, &ring, ring_flags);
|
145
334
|
if (ret)
|
146
|
-
|
335
|
+
t_error(1, ret, "io_uring: queue init");
|
336
|
+
|
337
|
+
set_cpu_affinity();
|
338
|
+
set_iowq_affinity(&ring);
|
147
339
|
|
148
340
|
if (cfg_fixed_files) {
|
149
341
|
ret = io_uring_register_files(&ring, &fd, 1);
|
150
342
|
if (ret < 0)
|
151
|
-
|
343
|
+
t_error(1, ret, "io_uring: files registration");
|
152
344
|
}
|
153
345
|
if (cfg_reg_ringfd) {
|
154
346
|
ret = io_uring_register_ring_fd(&ring);
|
155
347
|
if (ret < 0)
|
156
|
-
|
348
|
+
t_error(1, ret, "io_uring: io_uring_register_ring_fd");
|
157
349
|
}
|
158
350
|
|
159
351
|
iov.iov_base = payload;
|
@@ -161,7 +353,9 @@ static void do_tx(int domain, int type, int protocol)
|
|
161
353
|
|
162
354
|
ret = io_uring_register_buffers(&ring, &iov, 1);
|
163
355
|
if (ret)
|
164
|
-
|
356
|
+
t_error(1, ret, "io_uring: buffer registration");
|
357
|
+
|
358
|
+
pthread_barrier_wait(&barrier);
|
165
359
|
|
166
360
|
tstop = gettimeofday_ms() + cfg_runtime_ms;
|
167
361
|
do {
|
@@ -191,16 +385,20 @@ static void do_tx(int domain, int type, int protocol)
|
|
191
385
|
}
|
192
386
|
}
|
193
387
|
|
194
|
-
|
388
|
+
if (cfg_defer_taskrun && compl_cqes >= notif_slack)
|
389
|
+
ret = io_uring_submit_and_get_events(&ring);
|
390
|
+
else
|
391
|
+
ret = io_uring_submit(&ring);
|
392
|
+
|
195
393
|
if (ret != cfg_nr_reqs)
|
196
|
-
|
394
|
+
t_error(1, ret, "submit");
|
197
395
|
|
198
396
|
for (i = 0; i < cfg_nr_reqs; i++) {
|
199
397
|
cqe = wait_cqe_fast(&ring);
|
200
398
|
|
201
399
|
if (cqe->flags & IORING_CQE_F_NOTIF) {
|
202
400
|
if (cqe->flags & IORING_CQE_F_MORE)
|
203
|
-
|
401
|
+
t_error(1, -EINVAL, "F_MORE notif");
|
204
402
|
compl_cqes--;
|
205
403
|
i--;
|
206
404
|
io_uring_cqe_seen(&ring, cqe);
|
@@ -210,14 +408,14 @@ static void do_tx(int domain, int type, int protocol)
|
|
210
408
|
compl_cqes++;
|
211
409
|
|
212
410
|
if (cqe->res >= 0) {
|
213
|
-
packets++;
|
214
|
-
bytes += cqe->res;
|
411
|
+
td->packets++;
|
412
|
+
td->bytes += cqe->res;
|
215
413
|
} else if (cqe->res == -ECONNREFUSED || cqe->res == -EPIPE ||
|
216
414
|
cqe->res == -ECONNRESET) {
|
217
|
-
fprintf(stderr, "Connection failure");
|
415
|
+
fprintf(stderr, "Connection failure\n");
|
218
416
|
goto out_fail;
|
219
417
|
} else if (cqe->res != -EAGAIN) {
|
220
|
-
|
418
|
+
t_error(1, cqe->res, "send failed");
|
221
419
|
}
|
222
420
|
io_uring_cqe_seen(&ring, cqe);
|
223
421
|
}
|
@@ -226,12 +424,7 @@ static void do_tx(int domain, int type, int protocol)
|
|
226
424
|
out_fail:
|
227
425
|
shutdown(fd, SHUT_RDWR);
|
228
426
|
if (close(fd))
|
229
|
-
|
230
|
-
|
231
|
-
fprintf(stderr, "tx=%lu (MB=%lu), tx/s=%lu (MB/s=%lu)\n",
|
232
|
-
packets, bytes >> 20,
|
233
|
-
packets / (cfg_runtime_ms / 1000),
|
234
|
-
(bytes >> 20) / (cfg_runtime_ms / 1000));
|
427
|
+
t_error(1, errno, "close");
|
235
428
|
|
236
429
|
while (compl_cqes) {
|
237
430
|
struct io_uring_cqe *cqe = wait_cqe_fast(&ring);
|
@@ -242,25 +435,28 @@ out_fail:
|
|
242
435
|
io_uring_queue_exit(&ring);
|
243
436
|
}
|
244
437
|
|
245
|
-
|
438
|
+
|
439
|
+
static void *do_test(void *arg)
|
246
440
|
{
|
247
|
-
|
441
|
+
struct thread_data *td = arg;
|
442
|
+
int protocol = 0;
|
248
443
|
|
249
|
-
|
250
|
-
payload[i] = 'a' + (i % 26);
|
444
|
+
setup_sockaddr(cfg_family, str_addr, &td->dst_addr);
|
251
445
|
|
252
|
-
do_tx(
|
446
|
+
do_tx(td, cfg_family, cfg_type, protocol);
|
447
|
+
pthread_exit(&td->ret);
|
448
|
+
return NULL;
|
253
449
|
}
|
254
450
|
|
255
451
|
static void usage(const char *filepath)
|
256
452
|
{
|
257
|
-
|
453
|
+
t_error(1, 0, "Usage: %s [-n<N>] [-z<val>] [-s<payload size>] "
|
258
454
|
"(-4|-6) [-t<time s>] -D<dst_ip> udp", filepath);
|
259
455
|
}
|
260
456
|
|
261
457
|
static void parse_opts(int argc, char **argv)
|
262
458
|
{
|
263
|
-
const int max_payload_len =
|
459
|
+
const int max_payload_len = IP_MAXPACKET -
|
264
460
|
sizeof(struct ipv6hdr) -
|
265
461
|
sizeof(struct tcphdr) -
|
266
462
|
40 /* max tcp options */;
|
@@ -272,17 +468,17 @@ static void parse_opts(int argc, char **argv)
|
|
272
468
|
|
273
469
|
cfg_payload_len = max_payload_len;
|
274
470
|
|
275
|
-
while ((c = getopt(argc, argv, "46D:p:s:t:n:z:b:
|
471
|
+
while ((c = getopt(argc, argv, "46D:p:s:t:n:z:b:l:dC:T:R")) != -1) {
|
276
472
|
switch (c) {
|
277
473
|
case '4':
|
278
474
|
if (cfg_family != PF_UNSPEC)
|
279
|
-
|
475
|
+
t_error(1, 0, "Pass one of -4 or -6");
|
280
476
|
cfg_family = PF_INET;
|
281
477
|
cfg_alen = sizeof(struct sockaddr_in);
|
282
478
|
break;
|
283
479
|
case '6':
|
284
480
|
if (cfg_family != PF_UNSPEC)
|
285
|
-
|
481
|
+
t_error(1, 0, "Pass one of -4 or -6");
|
286
482
|
cfg_family = PF_INET6;
|
287
483
|
cfg_alen = sizeof(struct sockaddr_in6);
|
288
484
|
break;
|
@@ -307,15 +503,32 @@ static void parse_opts(int argc, char **argv)
|
|
307
503
|
case 'b':
|
308
504
|
cfg_fixed_buf = strtoul(optarg, NULL, 0);
|
309
505
|
break;
|
506
|
+
case 'l':
|
507
|
+
cfg_hugetlb = strtoul(optarg, NULL, 0);
|
508
|
+
break;
|
509
|
+
case 'd':
|
510
|
+
cfg_defer_taskrun = 1;
|
511
|
+
break;
|
512
|
+
case 'C':
|
513
|
+
cfg_cpu = strtol(optarg, NULL, 0);
|
514
|
+
break;
|
515
|
+
case 'T':
|
516
|
+
cfg_nr_threads = strtol(optarg, NULL, 0);
|
517
|
+
if (cfg_nr_threads > MAX_THREADS)
|
518
|
+
t_error(1, 0, "too many threads\n");
|
519
|
+
break;
|
520
|
+
case 'R':
|
521
|
+
cfg_rx = 1;
|
522
|
+
break;
|
310
523
|
}
|
311
524
|
}
|
312
525
|
|
313
526
|
if (cfg_nr_reqs > MAX_SUBMIT_NR)
|
314
|
-
|
527
|
+
t_error(1, 0, "-n: submit batch nr exceeds max (%d)", MAX_SUBMIT_NR);
|
315
528
|
if (cfg_payload_len > max_payload_len)
|
316
|
-
|
529
|
+
t_error(1, 0, "-s: payload exceeds max (%d)", max_payload_len);
|
317
530
|
|
318
|
-
|
531
|
+
str_addr = daddr;
|
319
532
|
|
320
533
|
if (optind != argc - 1)
|
321
534
|
usage(argv[0]);
|
@@ -323,17 +536,63 @@ static void parse_opts(int argc, char **argv)
|
|
323
536
|
|
324
537
|
int main(int argc, char **argv)
|
325
538
|
{
|
539
|
+
unsigned long long packets = 0, bytes = 0;
|
540
|
+
struct thread_data *td;
|
326
541
|
const char *cfg_test;
|
542
|
+
unsigned int i;
|
543
|
+
void *res;
|
327
544
|
|
328
545
|
parse_opts(argc, argv);
|
546
|
+
set_cpu_affinity();
|
547
|
+
|
548
|
+
payload = payload_buf;
|
549
|
+
if (cfg_hugetlb) {
|
550
|
+
payload = mmap(NULL, 2*1024*1024, PROT_READ | PROT_WRITE,
|
551
|
+
MAP_PRIVATE | MAP_HUGETLB | MAP_HUGE_2MB | MAP_ANONYMOUS,
|
552
|
+
-1, 0);
|
553
|
+
if (payload == MAP_FAILED) {
|
554
|
+
fprintf(stderr, "hugetlb alloc failed\n");
|
555
|
+
return 1;
|
556
|
+
}
|
557
|
+
}
|
329
558
|
|
330
559
|
cfg_test = argv[argc - 1];
|
331
560
|
if (!strcmp(cfg_test, "tcp"))
|
332
|
-
|
561
|
+
cfg_type = SOCK_STREAM;
|
333
562
|
else if (!strcmp(cfg_test, "udp"))
|
334
|
-
|
563
|
+
cfg_type = SOCK_DGRAM;
|
335
564
|
else
|
336
|
-
|
565
|
+
t_error(1, 0, "unknown cfg_test %s", cfg_test);
|
566
|
+
|
567
|
+
pthread_barrier_init(&barrier, NULL, cfg_nr_threads);
|
568
|
+
|
569
|
+
for (i = 0; i < IP_MAXPACKET; i++)
|
570
|
+
payload[i] = 'a' + (i % 26);
|
571
|
+
|
572
|
+
for (i = 0; i < cfg_nr_threads; i++) {
|
573
|
+
td = &threads[i];
|
574
|
+
td->idx = i;
|
575
|
+
}
|
576
|
+
|
577
|
+
if (cfg_rx)
|
578
|
+
do_setup_rx(cfg_family, cfg_type, 0);
|
579
|
+
|
580
|
+
for (i = 0; i < cfg_nr_threads; i++)
|
581
|
+
pthread_create(&threads[i].thread, NULL,
|
582
|
+
!cfg_rx ? do_test : do_rx, &threads[i]);
|
583
|
+
|
584
|
+
for (i = 0; i < cfg_nr_threads; i++) {
|
585
|
+
td = &threads[i];
|
586
|
+
pthread_join(td->thread, &res);
|
587
|
+
packets += td->packets;
|
588
|
+
bytes += td->bytes;
|
589
|
+
}
|
590
|
+
|
591
|
+
fprintf(stderr, "packets=%llu (MB=%llu), rps=%llu (MB/s=%llu)\n",
|
592
|
+
packets, bytes >> 20,
|
593
|
+
packets / (cfg_runtime_ms / 1000),
|
594
|
+
(bytes >> 20) / (cfg_runtime_ms / 1000));
|
337
595
|
|
596
|
+
pthread_barrier_destroy(&barrier);
|
338
597
|
return 0;
|
339
598
|
}
|
@@ -68,23 +68,8 @@ DEFINE_AWAIT_OP(readv)
|
|
68
68
|
DEFINE_AWAIT_OP(writev)
|
69
69
|
#undef DEFINE_AWAIT_OP
|
70
70
|
|
71
|
-
int
|
72
|
-
|
73
|
-
struct io_uring_cqe *cqe;
|
74
|
-
if (!sqe)
|
75
|
-
return -1;
|
76
|
-
|
77
|
-
io_uring_prep_poll_add(sqe, fd, poll_mask);
|
78
|
-
io_uring_sqe_set_data(sqe, pctx);
|
79
|
-
swapcontext(&pctx->ctx_fnew, &pctx->ctx_main);
|
80
|
-
io_uring_peek_cqe(pctx->ring, &cqe);
|
81
|
-
assert(cqe);
|
82
|
-
io_uring_cqe_seen(pctx->ring, cqe);
|
83
|
-
|
84
|
-
return cqe->res;
|
85
|
-
}
|
86
|
-
|
87
|
-
int await_delay(async_context *pctx, time_t seconds) {
|
71
|
+
static int await_delay(async_context *pctx, time_t seconds)
|
72
|
+
{
|
88
73
|
struct io_uring_sqe *sqe = io_uring_get_sqe(pctx->ring);
|
89
74
|
struct io_uring_cqe *cqe;
|
90
75
|
struct __kernel_timespec ts = {
|
@@ -0,0 +1,12 @@
|
|
1
|
+
prefix=@prefix@
|
2
|
+
exec_prefix=${prefix}
|
3
|
+
libdir=@libdir@
|
4
|
+
includedir=@includedir@
|
5
|
+
|
6
|
+
Name: @NAME@
|
7
|
+
Version: @VERSION@
|
8
|
+
Description: io_uring FFI library
|
9
|
+
URL: https://git.kernel.dk/cgit/liburing/
|
10
|
+
|
11
|
+
Libs: -L${libdir} -luring-ffi
|
12
|
+
Cflags: -I${includedir}
|
@@ -13,13 +13,13 @@
|
|
13
13
|
# GNU General Public License for more details.
|
14
14
|
#
|
15
15
|
# You should have received a copy of the GNU General Public License
|
16
|
-
# along with this program. If not, see <
|
16
|
+
# along with this program. If not, see <https://www.gnu.org/licenses/>.
|
17
17
|
#
|
18
18
|
set -xe
|
19
19
|
|
20
20
|
# Create dir for build
|
21
21
|
base=${1:-/tmp/release}
|
22
|
-
|
22
|
+
distro=unstable
|
23
23
|
releasedir=$base/$(lsb_release -si)/liburing
|
24
24
|
rm -rf $releasedir
|
25
25
|
mkdir -p $releasedir
|
@@ -40,7 +40,7 @@ git clean -dxf
|
|
40
40
|
# Change changelog if it's needed
|
41
41
|
cur_ver=`head -l debian/changelog | sed -n -e 's/.* (\(.*\)) .*/\1/p'`
|
42
42
|
if [ "$cur_ver" != "$version-1" ]; then
|
43
|
-
dch -D $
|
43
|
+
dch -D $distro --force-distribution -b -v "$version-1" "new version"
|
44
44
|
fi
|
45
45
|
|
46
46
|
# Create tar archieve
|
@@ -0,0 +1 @@
|
|
1
|
+
io_uring_check_version.3
|
@@ -0,0 +1 @@
|
|
1
|
+
io_uring_check_version.3
|
@@ -0,0 +1 @@
|
|
1
|
+
io_uring_check_version.3
|
@@ -9,12 +9,12 @@ io_uring_buf_ring_add \- add buffers to a shared buffer ring
|
|
9
9
|
.nf
|
10
10
|
.B #include <liburing.h>
|
11
11
|
.PP
|
12
|
-
.BI "
|
13
|
-
.BI "
|
14
|
-
.BI "
|
15
|
-
.BI "
|
16
|
-
.BI "
|
17
|
-
.BI "
|
12
|
+
.BI "void io_uring_buf_ring_add(struct io_uring_buf_ring *" br ",
|
13
|
+
.BI " void *" addr ",
|
14
|
+
.BI " unsigned int " len ",
|
15
|
+
.BI " unsigned short " bid ",
|
16
|
+
.BI " int " mask ",
|
17
|
+
.BI " int " buf_offset ");"
|
18
18
|
.fi
|
19
19
|
.SH DESCRIPTION
|
20
20
|
.PP
|