polyphony 0.93 → 0.95

data/vendor/liburing/examples/send-zerocopy.c

@@ -0,0 +1,339 @@
+/* SPDX-License-Identifier: MIT */
+/* based on linux-kernel/tools/testing/selftests/net/msg_zerocopy.c */
+#include <stdio.h>
+#include <stdlib.h>
+#include <stdint.h>
+#include <assert.h>
+#include <errno.h>
+#include <error.h>
+#include <limits.h>
+#include <fcntl.h>
+#include <unistd.h>
+#include <stdbool.h>
+#include <string.h>
+
+#include <arpa/inet.h>
+#include <linux/errqueue.h>
+#include <linux/if_packet.h>
+#include <linux/ipv6.h>
+#include <linux/socket.h>
+#include <linux/sockios.h>
+#include <net/ethernet.h>
+#include <net/if.h>
+#include <netinet/ip.h>
+#include <netinet/in.h>
+#include <netinet/ip6.h>
+#include <netinet/tcp.h>
+#include <netinet/udp.h>
+#include <sys/socket.h>
+#include <sys/time.h>
+#include <sys/resource.h>
+#include <sys/un.h>
+#include <sys/ioctl.h>
+#include <sys/socket.h>
+#include <sys/stat.h>
+#include <sys/time.h>
+#include <sys/types.h>
+#include <sys/wait.h>
+
+#include "liburing.h"
+
+#define ZC_TAG 0xfffffffULL
+#define MAX_SUBMIT_NR 512
+
+static bool cfg_reg_ringfd = true;
+static bool cfg_fixed_files = 1;
+static bool cfg_zc = 1;
+static int  cfg_nr_reqs = 8;
+static bool cfg_fixed_buf = 1;
+
+static int  cfg_family		= PF_UNSPEC;
+static int  cfg_payload_len;
+static int  cfg_port		= 8000;
+static int  cfg_runtime_ms	= 4200;
+
+static socklen_t cfg_alen;
+static struct sockaddr_storage cfg_dst_addr;
+
+static char payload[IP_MAXPACKET] __attribute__((aligned(4096)));
+
+static unsigned long gettimeofday_ms(void)
+{
+	struct timeval tv;
+
+	gettimeofday(&tv, NULL);
+	return (tv.tv_sec * 1000) + (tv.tv_usec / 1000);
+}
+
+static void do_setsockopt(int fd, int level, int optname, int val)
+{
+	if (setsockopt(fd, level, optname, &val, sizeof(val)))
+		error(1, errno, "setsockopt %d.%d: %d", level, optname, val);
+}
+
+static void setup_sockaddr(int domain, const char *str_addr,
+			   struct sockaddr_storage *sockaddr)
+{
+	struct sockaddr_in6 *addr6 = (void *) sockaddr;
+	struct sockaddr_in *addr4 = (void *) sockaddr;
+
+	switch (domain) {
+	case PF_INET:
+		memset(addr4, 0, sizeof(*addr4));
+		addr4->sin_family = AF_INET;
+		addr4->sin_port = htons(cfg_port);
+		if (str_addr &&
+		    inet_pton(AF_INET, str_addr, &(addr4->sin_addr)) != 1)
+			error(1, 0, "ipv4 parse error: %s", str_addr);
+		break;
+	case PF_INET6:
+		memset(addr6, 0, sizeof(*addr6));
+		addr6->sin6_family = AF_INET6;
+		addr6->sin6_port = htons(cfg_port);
+		if (str_addr &&
+		    inet_pton(AF_INET6, str_addr, &(addr6->sin6_addr)) != 1)
+			error(1, 0, "ipv6 parse error: %s", str_addr);
+		break;
+	default:
+		error(1, 0, "illegal domain");
+	}
+}
+
+static int do_setup_tx(int domain, int type, int protocol)
+{
+	int fd;
+
+	fd = socket(domain, type, protocol);
+	if (fd == -1)
+		error(1, errno, "socket t");
+
+	do_setsockopt(fd, SOL_SOCKET, SO_SNDBUF, 1 << 21);
+
+	if (connect(fd, (void *) &cfg_dst_addr, cfg_alen))
+		error(1, errno, "connect");
+	return fd;
+}
+
+static inline struct io_uring_cqe *wait_cqe_fast(struct io_uring *ring)
+{
+	struct io_uring_cqe *cqe;
+	unsigned head;
+	int ret;
+
+	io_uring_for_each_cqe(ring, head, cqe)
+		return cqe;
+
+	ret = io_uring_wait_cqe(ring, &cqe);
+	if (ret)
+		error(1, ret, "wait cqe");
+	return cqe;
+}
+
+static void do_tx(int domain, int type, int protocol)
+{
+	unsigned long packets = 0;
+	unsigned long bytes = 0;
+	struct io_uring ring;
+	struct iovec iov;
+	uint64_t tstop;
+	int i, fd, ret;
+	int compl_cqes = 0;
+
+	fd = do_setup_tx(domain, type, protocol);
+
+	ret = io_uring_queue_init(512, &ring, IORING_SETUP_COOP_TASKRUN);
+	if (ret)
+		error(1, ret, "io_uring: queue init");
+
+	if (cfg_fixed_files) {
+		ret = io_uring_register_files(&ring, &fd, 1);
+		if (ret < 0)
+			error(1, ret, "io_uring: files registration");
+	}
+	if (cfg_reg_ringfd) {
+		ret = io_uring_register_ring_fd(&ring);
+		if (ret < 0)
+			error(1, ret, "io_uring: io_uring_register_ring_fd");
+	}
+
+	iov.iov_base = payload;
+	iov.iov_len = cfg_payload_len;
+
+	ret = io_uring_register_buffers(&ring, &iov, 1);
+	if (ret)
+		error(1, ret, "io_uring: buffer registration");
+
+	tstop = gettimeofday_ms() + cfg_runtime_ms;
+	do {
+		struct io_uring_sqe *sqe;
+		struct io_uring_cqe *cqe;
+		unsigned buf_idx = 0;
+		unsigned msg_flags = MSG_WAITALL;
+
+		for (i = 0; i < cfg_nr_reqs; i++) {
+			sqe = io_uring_get_sqe(&ring);
+
+			if (!cfg_zc)
+				io_uring_prep_send(sqe, fd, payload,
+						   cfg_payload_len, 0);
+			else {
+				io_uring_prep_send_zc(sqe, fd, payload,
+						     cfg_payload_len, msg_flags, 0);
+				if (cfg_fixed_buf) {
+					sqe->ioprio |= IORING_RECVSEND_FIXED_BUF;
+					sqe->buf_index = buf_idx;
+				}
+			}
+			sqe->user_data = 1;
+			if (cfg_fixed_files) {
+				sqe->fd = 0;
+				sqe->flags |= IOSQE_FIXED_FILE;
+			}
+		}
+
+		ret = io_uring_submit(&ring);
+		if (ret != cfg_nr_reqs)
+			error(1, ret, "submit");
+
+		for (i = 0; i < cfg_nr_reqs; i++) {
+			cqe = wait_cqe_fast(&ring);
+
+			if (cqe->flags & IORING_CQE_F_NOTIF) {
+				if (cqe->flags & IORING_CQE_F_MORE)
+					error(1, -EINVAL, "F_MORE notif");
+				compl_cqes--;
+				i--;
+				io_uring_cqe_seen(&ring, cqe);
+				continue;
+			}
+			if (cqe->flags & IORING_CQE_F_MORE)
+				compl_cqes++;
+
+			if (cqe->res >= 0) {
+				packets++;
+				bytes += cqe->res;
+			} else if (cqe->res == -ECONNREFUSED || cqe->res == -EPIPE ||
+				   cqe->res == -ECONNRESET) {
+				fprintf(stderr, "Connection failure");
+				goto out_fail;
+			} else if (cqe->res != -EAGAIN) {
+				error(1, cqe->res, "send failed");
+			}
+			io_uring_cqe_seen(&ring, cqe);
+		}
+	} while (gettimeofday_ms() < tstop);
+
+out_fail:
+	shutdown(fd, SHUT_RDWR);
+	if (close(fd))
+		error(1, errno, "close");
+
+	fprintf(stderr, "tx=%lu (MB=%lu), tx/s=%lu (MB/s=%lu)\n",
+			packets, bytes >> 20,
+			packets / (cfg_runtime_ms / 1000),
+			(bytes >> 20) / (cfg_runtime_ms / 1000));
+
+	while (compl_cqes) {
+		struct io_uring_cqe *cqe = wait_cqe_fast(&ring);
+
+		io_uring_cqe_seen(&ring, cqe);
+		compl_cqes--;
+	}
+	io_uring_queue_exit(&ring);
+}
+
+static void do_test(int domain, int type, int protocol)
+{
+	int i;
+
+	for (i = 0; i < IP_MAXPACKET; i++)
+		payload[i] = 'a' + (i % 26);
+
+	do_tx(domain, type, protocol);
+}
+
+static void usage(const char *filepath)
+{
+	error(1, 0, "Usage: %s [-n<N>] [-z<val>] [-s<payload size>] "
+		    "(-4|-6) [-t<time s>] -D<dst_ip> udp", filepath);
+}
+
+static void parse_opts(int argc, char **argv)
+{
+	const int max_payload_len = sizeof(payload) -
+				    sizeof(struct ipv6hdr) -
+				    sizeof(struct tcphdr) -
+				    40 /* max tcp options */;
+	int c;
+	char *daddr = NULL;
+
+	if (argc <= 1)
+		usage(argv[0]);
+
+	cfg_payload_len = max_payload_len;
+
+	while ((c = getopt(argc, argv, "46D:p:s:t:n:z:b:k")) != -1) {
+		switch (c) {
+		case '4':
+			if (cfg_family != PF_UNSPEC)
+				error(1, 0, "Pass one of -4 or -6");
+			cfg_family = PF_INET;
+			cfg_alen = sizeof(struct sockaddr_in);
+			break;
+		case '6':
+			if (cfg_family != PF_UNSPEC)
+				error(1, 0, "Pass one of -4 or -6");
+			cfg_family = PF_INET6;
+			cfg_alen = sizeof(struct sockaddr_in6);
+			break;
+		case 'D':
+			daddr = optarg;
+			break;
+		case 'p':
+			cfg_port = strtoul(optarg, NULL, 0);
+			break;
+		case 's':
+			cfg_payload_len = strtoul(optarg, NULL, 0);
+			break;
+		case 't':
+			cfg_runtime_ms = 200 + strtoul(optarg, NULL, 10) * 1000;
+			break;
+		case 'n':
+			cfg_nr_reqs = strtoul(optarg, NULL, 0);
+			break;
+		case 'z':
+			cfg_zc = strtoul(optarg, NULL, 0);
+			break;
+		case 'b':
+			cfg_fixed_buf = strtoul(optarg, NULL, 0);
+			break;
+		}
+	}
+
+	if (cfg_nr_reqs > MAX_SUBMIT_NR)
+		error(1, 0, "-n: submit batch nr exceeds max (%d)", MAX_SUBMIT_NR);
+	if (cfg_payload_len > max_payload_len)
+		error(1, 0, "-s: payload exceeds max (%d)", max_payload_len);
+
+	setup_sockaddr(cfg_family, daddr, &cfg_dst_addr);
+
+	if (optind != argc - 1)
+		usage(argv[0]);
+}
+
+int main(int argc, char **argv)
+{
+	const char *cfg_test;
+
+	parse_opts(argc, argv);
+
+	cfg_test = argv[argc - 1];
+	if (!strcmp(cfg_test, "tcp"))
+		do_test(cfg_family, SOCK_STREAM, 0);
+	else if (!strcmp(cfg_test, "udp"))
+		do_test(cfg_family, SOCK_DGRAM, 0);
+	else
+		error(1, 0, "unknown cfg_test %s", cfg_test);
+
+	return 0;
+}

data/vendor/liburing/liburing.spec

@@ -1,5 +1,5 @@
 Name: liburing
-Version: 2.2
+Version: 2.3
 Release: 1%{?dist}
 Summary: Linux-native io_uring I/O access library
 License: (GPLv2 with exceptions and LGPLv2+) or MIT

data/vendor/liburing/man/io_uring.7

@@ -2,7 +2,7 @@
 .\" SPDX-License-Identifier: LGPL-2.0-or-later
 .\"
 
-.TH IO_URING 7 2020-07-26 "Linux" "Linux Programmer's Manual"
+.TH io_uring 7 2020-07-26 "Linux" "Linux Programmer's Manual"
 .SH NAME
 io_uring \- Asynchronous I/O facility
 .SH SYNOPSIS
@@ -277,7 +277,8 @@ you need to acquire a submission queue entry (SQE) from the submission
 queue (SQ),
 fill it up with details of the operation you want to submit and call 
 .BR io_uring_enter (2). 
-If you want to avoid calling 
+There are helper functions of the form io_uring_prep_X to enable proper
+setup of the SQE. If you want to avoid calling
 .BR io_uring_enter (2),
 you have the option of setting up Submission Queue Polling.
 .PP
@@ -404,10 +405,36 @@ this field can be used to correlate a submission with a completion.
 is the result from the system call that was performed as part of the
 submission;
 its return value.
+
 The
 .I flags
-field could carry request-specific metadata in the future,
-but is currently unused.
+field carries request-specific information. As of the 6.0 kernel, the following
+flags are defined:
+
+.TP
+.B IORING_CQE_F_BUFFER
+If set, the upper 16 bits of the flags field carries the buffer ID that was
+chosen for this request. The request must have been issued with
+.B IOSQE_BUFFER_SELECT
+set, and used with a request type that supports buffer selection. Additionally,
+buffers must have been provided upfront either via the
+.B IORING_OP_PROVIDE_BUFFERS
+or the
+.B IORING_REGISTER_PBUF_RING
+methods.
+.TP
+.B IORING_CQE_F_MORE
+If set, the application should expect more completions from the request. This
+is used for requests that can generate multiple completions, such as multi-shot
+requests, receive, or accept.
+.TP
+.B IORING_CQE_F_SOCK_NONEMPTY
+If set, upon receiving the data from the socket in the current request, the
+socket still had data left on completion of this request.
+.TP
+.B IORING_CQE_F_NOTIF
+Set for notification CQEs, as seen with the zero-copy networking send and
+receive support.
 .PP
 The general sequence to read completion events off the completion queue is
 as follows:
@@ -484,7 +511,7 @@ them to the submission queue. This avoids the
 .BR io_uring_enter (2)
 call you need to make to tell the kernel to pick SQEs up.
 For high-performance applications,
-this means even lesser system call overheads.
+this means even fewer system call overheads.
 .SH CONFORMING TO
 .B io_uring
 is Linux-specific.
@@ -551,8 +578,8 @@ int io_uring_setup(unsigned entries, struct io_uring_params *p)
 int io_uring_enter(int ring_fd, unsigned int to_submit,
                    unsigned int min_complete, unsigned int flags)
 {
-    return (int) syscall(__NR_io_uring_enter, ring_fd, to_submit, min_complete,
-                         flags, NULL, 0);
+    return (int) syscall(__NR_io_uring_enter, ring_fd, to_submit,
+    			 min_complete, flags, NULL, 0);
 }
 
 int app_setup_uring(void) {
@@ -696,10 +723,10 @@ int submit_to_sq(int fd, int op) {
     io_uring_smp_store_release(sring_tail, tail);
 
     /*
-    * Tell the kernel we have submitted events with the io_uring_enter() system
-    * call. We also pass in the IOURING_ENTER_GETEVENTS flag which causes the
-    * io_uring_enter() call to wait until min_complete (the 3rd param) events
-    * complete.
+    * Tell the kernel we have submitted events with the io_uring_enter()
+    * system call. We also pass in the IOURING_ENTER_GETEVENTS flag which
+    * causes the io_uring_enter() call to wait until min_complete
+    * (the 3rd param) events complete.
     * */
     int ret =  io_uring_enter(ring_fd, 1,1,
                               IORING_ENTER_GETEVENTS);

data/vendor/liburing/man/io_uring_buf_ring_add.3

@@ -0,0 +1,53 @@
+.\" Copyright (C) 2022 Jens Axboe <axboe@kernel.dk>
+.\"
+.\" SPDX-License-Identifier: LGPL-2.0-or-later
+.\"
+.TH io_uring_buf_ring_add 3 "May 18, 2022" "liburing-2.2" "liburing Manual"
+.SH NAME
+io_uring_buf_ring_add \- add buffers to a shared buffer ring
+.SH SYNOPSIS
+.nf
+.B #include <liburing.h>
+.PP
+.BI "int io_uring_buf_ring_add(struct io_uring_buf_ring *" br ",
+.BI "                          void *" addr ",
+.BI "                          unsigned int " len ",
+.BI "                          unsigned short " bid ",
+.BI "                          int " mask ",
+.BI "                          int " buf_offset ");"
+.fi
+.SH DESCRIPTION
+.PP
+The
+.BR io_uring_buf_ring_add (3)
+adds a new buffer to the shared buffer ring
+.IR br .
+The buffer address is indicated by
+.I addr
+and is of
+.I len
+bytes of length.
+.I bid
+is the buffer ID, which will be returned in the CQE.
+.I mask
+is the size mask of the ring, available from
+.BR io_uring_buf_ring_mask (3) .
+.I buf_offset
+is the offset to insert at from the current tail. If just one buffer is provided
+before the ring tail is committed with
+.BR io_uring_buf_ring_advance (3)
+or
+.BR io_uring_buf_ring_cq_advance (3),
+then
+.I buf_offset
+should be 0. If buffers are provided in a loop before being committed, the
+.I buf_offset
+must be incremented by one for each buffer added.
+
+.SH RETURN VALUE
+None
+.SH SEE ALSO
+.BR io_uring_register_buf_ring (3),
+.BR io_uring_buf_ring_mask (3),
+.BR io_uring_buf_ring_advance (3),
+.BR io_uring_buf_ring_cq_advance (3)

data/vendor/liburing/man/io_uring_buf_ring_advance.3

@@ -0,0 +1,31 @@
+.\" Copyright (C) 2022 Jens Axboe <axboe@kernel.dk>
+.\"
+.\" SPDX-License-Identifier: LGPL-2.0-or-later
+.\"
+.TH io_uring_buf_ring_advance 3 "May 18, 2022" "liburing-2.2" "liburing Manual"
+.SH NAME
+io_uring_buf_ring_advance \- advance index of provided buffer in buffer ring
+.SH SYNOPSIS
+.nf
+.B #include <liburing.h>
+.PP
+.BI "void io_uring_buf_ring_advance(struct io_uring_buf_ring *" br ",
+.BI "                               int " count ");"
+.fi
+.SH DESCRIPTION
+.PP
+The
+.BR io_uring_buf_ring_advance (3)
+commits
+.I count
+previously added buffers to the shared buffer ring
+.IR br ,
+making them visible to the kernel and hence consumable. This passes ownership
+of the buffer to the ring.
+
+.SH RETURN VALUE
+None
+.SH SEE ALSO
+.BR io_uring_register_buf_ring (3),
+.BR io_uring_buf_ring_add (3),
+.BR io_uring_buf_ring_cq_advance (3)

data/vendor/liburing/man/io_uring_buf_ring_cq_advance.3

@@ -0,0 +1,41 @@
+.\" Copyright (C) 2022 Jens Axboe <axboe@kernel.dk>
+.\"
+.\" SPDX-License-Identifier: LGPL-2.0-or-later
+.\"
+.TH io_uring_buf_ring_cq_advance 3 "May 18, 2022" "liburing-2.2" "liburing Manual"
+.SH NAME
+io_uring_buf_ring_cq_advance \- advance index of provided buffer and CQ ring
+.SH SYNOPSIS
+.nf
+.B #include <liburing.h>
+.PP
+.BI "void io_uring_buf_ring_cq_advance(struct io_uring *" ring ",
+.BI "                                  struct io_uring_buf_ring *" br ",
+.BI "                                  int " count ");"
+.fi
+.SH DESCRIPTION
+.PP
+The
+.BR io_uring_buf_ring_cq_advance (3)
+commits
+.I count
+previously added buffers to the shared buffer ring
+.IR br ,
+making them visible to the kernel and hence consumable. This passes ownership
+of the buffer to the ring. At the same time, it advances the CQ ring of
+.I ring
+by
+.I count
+amount. This effectively bundles both a
+.BR io_uring_buf_ring_advance (3)
+call and a
+.BR io_uring_cq_avance (3)
+into one operation. Since updating either ring index entails a store memory
+barrier, doing both at once is more efficient.
+
+.SH RETURN VALUE
+None
+.SH SEE ALSO
+.BR io_uring_register_buf_ring (3),
+.BR io_uring_buf_ring_add (3),
+.BR io_uring_buf_ring_advance (3)

data/vendor/liburing/man/io_uring_buf_ring_init.3

@@ -0,0 +1,30 @@
+.\" Copyright (C) 2022 Dylan Yudaken <dylany@fb.com>
+.\"
+.\" SPDX-License-Identifier: LGPL-2.0-or-later
+.\"
+.TH io_uring_buf_ring_init 3 "June 13, 2022" "liburing-2.2" "liburing Manual"
+.SH NAME
+io_uring_buf_ring_init \- Initialise a  buffer ring
+.SH SYNOPSIS
+.nf
+.B #include <liburing.h>
+.PP
+.BI "void io_uring_buf_ring_init(struct io_uring_buf_ring *" br ");"
+.fi
+.SH DESCRIPTION
+.PP
+.BR io_uring_buf_ring_init (3)
+initialises
+.IR br
+so that it is ready to be used. It may be called after
+.BR io_uring_register_buf_ring (3)
+but must be called before the buffer ring is used in any other way.
+
+.SH RETURN VALUE
+None
+
+.SH SEE ALSO
+.BR io_uring_register_buf_ring (3),
+.BR io_uring_buf_ring_add (3)
+.BR io_uring_buf_ring_advance (3),
+.BR io_uring_buf_ring_cq_advance (3)

data/vendor/liburing/man/io_uring_buf_ring_mask.3

@@ -0,0 +1,27 @@
+.\" Copyright (C) 2022 Dylan Yudaken <dylany@fb.com>
+.\"
+.\" SPDX-License-Identifier: LGPL-2.0-or-later
+.\"
+.TH io_uring_buf_ring_mask 3 "June 13, 2022" "liburing-2.2" "liburing Manual"
+.SH NAME
+io_uring_buf_ring_mask \- Calculate buffer ring mask size
+.SH SYNOPSIS
+.nf
+.B #include <liburing.h>
+.PP
+.BI "int io_uring_buf_ring_mask(__u32 " ring_entries ");"
+.fi
+.SH DESCRIPTION
+.PP
+.BR io_uring_buf_ring_mask (3)
+calculates the appropriate size mask for a buffer ring.
+.IR ring_entries
+is the ring entries as specified in
+.BR io_uring_register_buf_ring (3) .
+
+.SH RETURN VALUE
+Size mask for the buffer ring.
+
+.SH SEE ALSO
+.BR io_uring_register_buf_ring (3),
+.BR io_uring_buf_ring_add (3)

data/vendor/liburing/man/io_uring_cq_advance.3

@@ -4,32 +4,46 @@
 .\"
 .TH io_uring_cq_advance 3 "January 25, 2022" "liburing-2.1" "liburing Manual"
 .SH NAME
-io_uring_cq_advance - Mark one or more io_uring completion events as processed
+io_uring_cq_advance \- mark one or more io_uring completion events as consumed
 .SH SYNOPSIS
 .nf
-.BR "#include <liburing.h>"
+.B #include <liburing.h>
 .PP
-.BI "void io_uring_cqe_advance(struct io_uring *ring,"
-.BI "                          unsigned nr)"
+.BI "void io_uring_cq_advance(struct io_uring *" ring ","
+.BI "                          unsigned " nr ");"
 .fi
-.PP
 .SH DESCRIPTION
 .PP
-The io_uring_cq_advance() function marks
+The
+.BR io_uring_cq_advance (3)
+function marks
 .I nr
-IO completions
-belonging to the
+IO completions belonging to the
 .I ring
-param as processed.
+param as consumed.
+
+After the caller has submitted a request with
+.BR io_uring_submit (3),
+the application can retrieve the completion with
+.BR io_uring_wait_cqe (3),
+.BR io_uring_peek_cqe (3),
+or any of the other CQE retrieval helpers, and mark it as consumed with
+.BR io_uring_cqe_seen (3).
 
-After the caller has submitted a request with io_uring_submit(), he can retrieve
-the completion with io_uring_wait_cqe() and mark it then as processed with
-io_uring_cqe_seen(). 
+The function
+.BR io_uring_cqe_seen (3)
+calls the function
+.BR io_uring_cq_advance (3).
 
-The function io_uring_cqe_seen() calls the function io_uring_cq_advance().
+Completions must be marked as seen, so their slot can get reused. Failure to do
+so will result in the same completion being returned on the next invocation.
 
-Completions must be marked as completed, so their slot can get reused.
 .SH RETURN VALUE
 None
 .SH SEE ALSO
-.BR io_uring_submit (3), io_uring_wait_cqe (3), io_uring_cqe_seen (3)
+.BR io_uring_submit (3),
+.BR io_uring_wait_cqe (3),
+.BR io_uring_peek_cqe (3),
+.BR io_uring_wait_cqes (3),
+.BR io_uring_wait_cqe_timeout (3),
+.BR io_uring_cqe_seen (3)