uringmachine 0.19.1 → 0.21.0

data/vendor/liburing/test/pipe.c

@@ -0,0 +1,166 @@
+// SPDX-License-Identifier: MIT
+/*
+ * Description: test pipe creation through io_uring
+ *
+ */
+#include <stdio.h>
+#include <string.h>
+#include <unistd.h>
+
+#include "helpers.h"
+#include "liburing.h"
+
+static int no_pipe;
+
+struct params {
+	int fixed;
+	int async;
+	int too_small;
+};
+
+static int pipe_comms(struct io_uring *ring, int *fds, struct params *p)
+{
+	struct io_uring_sqe *sqe;
+	struct io_uring_cqe *cqe;
+	char src[32], dst[32];
+	int i, ret;
+
+	memset(src, 0x5a, sizeof(src));
+	memset(dst, 0, sizeof(dst));
+
+	sqe = io_uring_get_sqe(ring);
+	io_uring_prep_write(sqe, fds[1], src, sizeof(src), 0);
+	if (p->fixed)
+		sqe->flags |= IOSQE_FIXED_FILE;
+	sqe->user_data = 1;
+	io_uring_submit(ring);
+
+	sqe = io_uring_get_sqe(ring);
+	io_uring_prep_read(sqe, fds[0], dst, sizeof(dst), 0);
+	if (p->fixed)
+		sqe->flags |= IOSQE_FIXED_FILE;
+	sqe->user_data = 2;
+	io_uring_submit(ring);
+
+	for (i = 0; i < 2; i++) {
+		ret = io_uring_wait_cqe(ring, &cqe);
+		if (ret) {
+			fprintf(stderr, "wait cqe %d\n", ret);
+			return 1;
+		}
+		if (cqe->res != 32) {
+			printf("ud=%d, res=%d\n", (int) cqe->user_data, cqe->res);
+			return 1;
+		}
+		io_uring_cqe_seen(ring, cqe);
+	}
+
+	return memcmp(dst, src, sizeof(src));
+}
+
+static int pipe_test(int init_flags, struct params *p)
+{
+	struct io_uring ring;
+	struct io_uring_sqe *sqe;
+	struct io_uring_cqe *cqe;
+	int ret, fds[2];
+
+	ret = io_uring_queue_init(8, &ring, init_flags);
+	/* can hit -ENOMEM due to repeated ring creation and teardowns */
+	if (ret == -ENOMEM) {
+		usleep(1000);
+		return 0;
+	} else if (ret) {
+		fprintf(stderr, "ring_init: %d\n", ret);
+		return 1;
+	}
+
+	if (p->fixed) {
+		int sz;
+
+		if (p->too_small)
+			sz = 1;
+		else
+			sz = 100;
+		ret = io_uring_register_files_sparse(&ring, sz);
+		if (ret) {
+			if (ret == -EINVAL) {
+				no_pipe = 1;
+				return 0;
+			}
+			fprintf(stderr, "Failed to register sparse table %d\n", ret);
+			return 1;
+		}
+	}
+
+	fds[0] = fds[1] = -1;
+	sqe = io_uring_get_sqe(&ring);
+	if (!p->fixed)
+		io_uring_prep_pipe(sqe, fds, 0);
+	else
+		io_uring_prep_pipe_direct(sqe, fds, 0, IORING_FILE_INDEX_ALLOC);
+	if (p->async)
+		sqe->flags |= IOSQE_ASYNC;
+
+	io_uring_submit(&ring);
+	ret = io_uring_wait_cqe(&ring, &cqe);
+	if (ret) {
+		fprintf(stderr, "wait: %d\n", ret);
+		return 1;
+	}
+	if (cqe->res) {
+		if (cqe->res == -EINVAL) {
+			no_pipe = 1;
+			return 0;
+		}
+		if (p->fixed && p->too_small && cqe->res == -ENFILE)
+			goto done;
+		fprintf(stderr, "Bad cqe res %d\n", cqe->res);
+		return 1;
+	}
+
+	io_uring_cqe_seen(&ring, cqe);
+
+	ret = pipe_comms(&ring, fds, p);
+	if (!p->fixed) {
+		close(fds[0]);
+		close(fds[1]);
+	}
+done:
+	io_uring_queue_exit(&ring);
+	return ret;
+}
+
+int main(int argc, char *argv[])
+{
+	int init_flags[] = { 0, IORING_SETUP_SQPOLL, IORING_SETUP_DEFER_TASKRUN | IORING_SETUP_SINGLE_ISSUER };
+	struct params ps[] = {
+		{ 0, 0, 0 },
+		{ 0, 1, 0 },
+		{ 1, 0, 0 },
+		{ 1, 1, 0 },
+		{ 0, 0, 1 },
+		{ 0, 1, 1 },
+		{ 1, 0, 1 },
+		{ 1, 1, 1 } };
+
+	int i, j;
+
+	if (argc > 1)
+		return T_EXIT_SKIP;
+
+	for (i = 0; i < ARRAY_SIZE(init_flags); i++) {
+		for (j = 0; j < ARRAY_SIZE(ps); j++) {
+			if (pipe_test(init_flags[i], &ps[j])) {
+				fprintf(stderr, "pipe %x %d/%d/%d failed\n",
+					init_flags[i], ps[j].fixed, ps[j].async,
+					ps[j].too_small);
+				return T_EXIT_FAIL;
+			}
+			if (no_pipe)
+				return T_EXIT_SKIP;
+		}
+	}
+
+	return T_EXIT_PASS;
+}

data/vendor/liburing/test/poll-race-mshot.c

@@ -158,7 +158,7 @@ static int test_mshot(struct io_uring *ring, struct data *d)
 
 	d->fd = fd[1];
 
-	if (posix_memalign((void *) &buf, 16384, BUF_SIZE * NREQS))
+	if (posix_memalign((void **) &buf, 16384, BUF_SIZE * NREQS))
 		return T_EXIT_FAIL;
 
 	br = io_uring_setup_buf_ring(ring, NREQS, 1, 0, &ret);
@@ -221,6 +221,18 @@ static int test_mshot(struct io_uring *ring, struct data *d)
 			fprintf(stderr, "Got too many requests?\n");
 			return T_EXIT_FAIL;
 		}
+		/*
+		 * We're using unix sockets, and later kernels got support added
+		 * for msg_inq querying. On those kernels, we cannot rely on
+		 * the multishot terminating on a zero receive, as io_uring
+		 * will not do that retry as it KNOWS there's zero bytes
+		 * pending. Hence we need to actively quiet at that point. Inc
+		 * 'i' as well as we don't get the non-MORE CQE posted.
+		 */
+		if (i == NREQS) {
+			i++;
+			break;
+		}
 	} while (1);
 
 	if (i != NREQS + 1) {

data/vendor/liburing/test/read-write.c

@@ -462,7 +462,7 @@ static int test_buf_select_short(const char *filename, int nonvec)
 	}
 
 	exp_len = 0;
-	for (i = 0; i < BUFFERS; i++) {
+	for (i = 0; i < 2 * BUFFERS; i++) {
 		sqe = io_uring_get_sqe(&ring);
 		io_uring_prep_provide_buffers(sqe, vecs[i].iov_base,
 						vecs[i].iov_len / 2, 1, 1, i);
@@ -471,12 +471,12 @@ static int test_buf_select_short(const char *filename, int nonvec)
 	}
 
 	ret = io_uring_submit(&ring);
-	if (ret != BUFFERS) {
+	if (ret != BUFFERS * 2) {
 		fprintf(stderr, "submit: %d\n", ret);
 		return -1;
 	}
 
-	for (i = 0; i < BUFFERS; i++) {
+	for (i = 0; i < BUFFERS * 2; i++) {
 		ret = io_uring_wait_cqe(&ring, &cqe);
 		if (cqe->res < 0) {
 			fprintf(stderr, "cqe->res=%d\n", cqe->res);
@@ -935,7 +935,7 @@ int main(int argc, char *argv[])
 
 	signal(SIGXFSZ, SIG_IGN);
 
-	vecs = t_create_buffers(BUFFERS, BS);
+	vecs = t_create_buffers(2 * BUFFERS, BS);
 
 	/* if we don't have nonvec read, skip testing that */
 	nr = has_nonvec_read() ? 64 : 32;

data/vendor/liburing/test/recv-mshot-fair.c

@@ -28,6 +28,7 @@
 
 #define NR_RDS	4
 
+static bool no_iter_support;
 static bool no_limit_support;
 
 static int use_port = PORT;
@@ -48,6 +49,7 @@ struct recv_data {
 	int total_bytes;
 
 	int recv_bundle;
+	int mshot_limit;
 
 	int use_port;
 	int id;
@@ -64,13 +66,11 @@ static void arm_recv(struct io_uring *ring, struct recv_data *rd)
 	struct io_uring_sqe *sqe;
 	int len = PER_ITER_LIMIT;
 
-	if (rd->total_bytes && rd->bytes_since_arm > PER_MSHOT_LIMIT)
-		rd->mshot_too_big++;
-
 	rd->bytes_since_arm = 0;
 	sqe = io_uring_get_sqe(ring);
 	io_uring_prep_recv_multishot(sqe, rd->accept_fd, NULL, len, 0);
-	sqe->optlen = PER_MSHOT_LIMIT;
+	if (rd->mshot_limit)
+		sqe->optlen = PER_MSHOT_LIMIT;
 	if (rd->recv_bundle)
 		sqe->ioprio |= IORING_RECVSEND_BUNDLE;
 	sqe->buf_group = RECV_BGID;
@@ -155,7 +155,7 @@ static int recv_get_cqe(struct io_uring *ring,
 	int ret;
 
 	do {
-		ret = io_uring_submit_and_wait_timeout(ring, cqe, 1, &ts, NULL);
+		ret = io_uring_wait_cqe_timeout(ring, cqe, &ts);
 		if (!ret) {
 			struct recv_data *rd = io_uring_cqe_get_data(*cqe);
 
@@ -196,6 +196,7 @@ static int do_recv(struct io_uring *ring)
 	struct recv_data *rd;
 	int ret = 1;
 	int done = 0;
+	int pending_submit = 0;
 
 	last_rd = NULL;
 	bytes_since_last = 0;
@@ -223,9 +224,15 @@ static int do_recv(struct io_uring *ring)
 			done++;
 			continue;
 		}
+		if (rd->mshot_limit && rd->bytes_since_arm > PER_MSHOT_LIMIT)
+			rd->mshot_too_big++;
 		io_uring_cqe_seen(ring, cqe);
-		if (!(cqe->flags & IORING_CQE_F_MORE) && rd->recv_bytes)
+		if (!(cqe->flags & IORING_CQE_F_MORE) && rd->recv_bytes) {
 			arm_recv(ring, rd);
+			pending_submit++;
+		}
+		if (pending_submit == NR_RDS)
+			io_uring_submit(ring);
 	} while (done < NR_RDS);
 
 	ret = 0;
@@ -241,7 +248,7 @@ static void *recv_fn(void *data)
 	struct io_uring_buf_ring *br = NULL;
 	struct io_uring ring;
 	unsigned int buf_len;
-	void *buf, *ptr;
+	void *buf = NULL, *ptr;
 	int ret, sock[NR_RDS], i;
 	int brflags, ring_setup = 0;
 
@@ -255,7 +262,7 @@ static void *recv_fn(void *data)
 	ret = io_uring_queue_init_params(16, &ring, &p);
 	if (ret) {
 		if (ret == -EINVAL) {
-			no_limit_support = true;
+			no_iter_support = true;
 			goto skip;
 		}
 		fprintf(stderr, "ring init: %d\n", ret);
@@ -270,7 +277,7 @@ static void *recv_fn(void *data)
 	br = io_uring_setup_buf_ring(&ring, RECV_BIDS, RECV_BGID, brflags, &ret);
 	if (!br) {
 		if (ret == -EINVAL) {
-			no_limit_support = true;
+			no_iter_support = true;
 			goto skip;
 		}
 		fprintf(stderr, "failed setting up recv ring %d\n", ret);
@@ -284,7 +291,7 @@ static void *recv_fn(void *data)
 		ptr += buf_len;
 	}
 	io_uring_buf_ring_advance(br, RECV_BIDS);
-	
+
 	for (i = 0; i < NR_RDS; i++) {
 		ret = recv_prep(&ring, &rds[i], &sock[i]);
 		if (ret) {
@@ -299,7 +306,10 @@ static void *recv_fn(void *data)
 
 		if (!io_uring_peek_cqe(&ring, &cqe)) {
 			if (cqe->res == -EINVAL) {
-				no_limit_support = true;
+				if (rds[0].mshot_limit)
+					no_limit_support = true;
+				else
+					no_iter_support = true;
 				goto skip;
 			}
 		}
@@ -319,6 +329,7 @@ skip:
 		io_uring_free_buf_ring(&ring, br, RECV_BIDS, RECV_BGID);
 	if (ring_setup)
 		io_uring_queue_exit(&ring);
+	free(buf);
 err:
 	return (void *)(intptr_t)ret;
 }
@@ -379,7 +390,8 @@ struct res {
 	int unfair;
 };
 
-static int test(int nr_send, int bundle, unsigned int queue_flags, struct res *r)
+static int test(int nr_send, int bundle, int mshot_limit,
+		unsigned int queue_flags, struct res *r)
 {
 	struct recv_data rds[NR_RDS] = { };
 	pthread_t recv_thread;
@@ -401,6 +413,7 @@ static int test(int nr_send, int bundle, unsigned int queue_flags, struct res *r
 		rd->max_sends = nr_send;
 		rd->recv_bundle = bundle;
 		rd->recv_bytes = nr_send * 4096;
+		rd->mshot_limit = mshot_limit;
 		rd->id = i + 1;
 	}
 
@@ -435,9 +448,9 @@ static int run_tests(void)
 	struct res r;
 	int ret;
 
-	ret = test(2, 1, IORING_SETUP_DEFER_TASKRUN | IORING_SETUP_SINGLE_ISSUER, &r);
+	ret = test(2, 1, 0, IORING_SETUP_DEFER_TASKRUN | IORING_SETUP_SINGLE_ISSUER, &r);
 	if (ret) {
-		if (no_limit_support)
+		if (no_iter_support)
 			return T_EXIT_SKIP;
 		fprintf(stderr, "test DEFER bundle failed\n");
 		return T_EXIT_FAIL;
@@ -449,52 +462,86 @@ static int run_tests(void)
 		return T_EXIT_FAIL;
 	}
 
-	ret = test(2, 0, IORING_SETUP_DEFER_TASKRUN | IORING_SETUP_SINGLE_ISSUER, &r);
+	ret = test(2, 0, 0, IORING_SETUP_DEFER_TASKRUN | IORING_SETUP_SINGLE_ISSUER, &r);
 	if (ret) {
 		fprintf(stderr, "test DEFER failed\n");
 		return T_EXIT_FAIL;
 	}
 
-	/* DEFER_TASKRUN should be fully fair and not have overshoots */
 	if (r.unfair || r.mshot_too_big) {
 		fprintf(stderr, "DEFER unfair=%d, too_big=%d\n", r.unfair, r.mshot_too_big);
 		return T_EXIT_FAIL;
 	}
 
-	ret = test(2, 1, IORING_SETUP_COOP_TASKRUN, &r);
+	ret = test(2, 1, 0, IORING_SETUP_COOP_TASKRUN, &r);
 	if (ret) {
 		fprintf(stderr, "test COOP bundle failed\n");
 		return T_EXIT_FAIL;
 	}
 
-	/*
-	 * normal task_work should not have overshoots, but may not be fair
-	 * because of the re-arming.
-	 */
-	if (r.unfair)
-		fprintf(stdout, "!DEFER bundle unfair, expected\n");
-	if (r.mshot_too_big) {
-		fprintf(stderr, "!DEFER bundle too_big=%d\n", r.mshot_too_big);
+	if (r.unfair || r.mshot_too_big) {
+		fprintf(stderr, "COOP bundle too_big=%d\n", r.mshot_too_big);
 		return T_EXIT_FAIL;
 	}
 
-	ret = test(2, 0, IORING_SETUP_COOP_TASKRUN, &r);
+	ret = test(2, 0, 0, IORING_SETUP_COOP_TASKRUN, &r);
 	if (ret) {
 		fprintf(stderr, "test COOP failed\n");
 		return T_EXIT_FAIL;
 	}
 
-	/*
-	 * normal task_work should not have overshoots, but may not be fair
-	 * because of the re-arming.
-	 */
-	if (r.unfair)
-		fprintf(stdout, "!DEFER unfair, expected\n");
-	if (r.mshot_too_big) {
+	if (r.unfair || r.mshot_too_big) {
 		fprintf(stderr, "!DEFER too_big=%d\n", r.mshot_too_big);
 		return T_EXIT_FAIL;
 	}
-	
+
+	ret = test(2, 1, 1, IORING_SETUP_DEFER_TASKRUN | IORING_SETUP_SINGLE_ISSUER, &r);
+	if (ret) {
+		if (no_limit_support)
+			return T_EXIT_PASS;
+		fprintf(stderr, "test DEFER bundle cap failed\n");
+		return T_EXIT_FAIL;
+	}
+
+	/* DEFER_TASKRUN should be fully fair and not have overshoots */
+	if (r.unfair || r.mshot_too_big) {
+		fprintf(stderr, "DEFER bundle cap unfair=%d, too_big=%d\n", r.unfair, r.mshot_too_big);
+		return T_EXIT_FAIL;
+	}
+
+	ret = test(2, 0, 1, IORING_SETUP_DEFER_TASKRUN | IORING_SETUP_SINGLE_ISSUER, &r);
+	if (ret) {
+		fprintf(stderr, "test DEFER cap failed\n");
+		return T_EXIT_FAIL;
+	}
+
+	if (r.unfair || r.mshot_too_big) {
+		fprintf(stderr, "DEFER cap unfair=%d, too_big=%d\n", r.unfair, r.mshot_too_big);
+		return T_EXIT_FAIL;
+	}
+
+	ret = test(2, 1, 1, IORING_SETUP_COOP_TASKRUN, &r);
+	if (ret) {
+		fprintf(stderr, "test COOP bundle cap failed\n");
+		return T_EXIT_FAIL;
+	}
+
+	if (r.unfair || r.mshot_too_big) {
+		fprintf(stderr, "COOP bundle cap too_big=%d\n", r.mshot_too_big);
+		return T_EXIT_FAIL;
+	}
+
+	ret = test(2, 0, 1, IORING_SETUP_COOP_TASKRUN, &r);
+	if (ret) {
+		fprintf(stderr, "test COOP cap failed\n");
+		return T_EXIT_FAIL;
+	}
+
+	if (r.unfair || r.mshot_too_big) {
+		fprintf(stderr, "COOP cap too_big=%d\n", r.mshot_too_big);
+		return T_EXIT_FAIL;
+	}
+
 	return T_EXIT_PASS;
 }
 

data/vendor/liburing/test/recvsend_bundle.c

@@ -21,7 +21,7 @@ static int nr_msgs;
 static int use_tcp;
 static int classic_buffers;
 
-#define RECV_BIDS	8192
+#define RECV_BIDS	16384
 #define RECV_BID_MASK	(RECV_BIDS - 1)
 
 #include "liburing.h"

data/vendor/liburing/test/resize-rings.c

@@ -657,6 +657,8 @@ try_defer:
 	ret = test(IORING_SETUP_SINGLE_ISSUER | IORING_SETUP_DEFER_TASKRUN, fd, 0);
 	if (ret == T_EXIT_FAIL)
 		return T_EXIT_FAIL;
+	else if (ret == T_EXIT_SKIP)
+		return T_EXIT_SKIP;
 
 	ret = test(IORING_SETUP_SINGLE_ISSUER | IORING_SETUP_DEFER_TASKRUN, fd, 1);
 	if (ret == T_EXIT_FAIL)