uringmachine 0.19.1 → 0.21.0

data/vendor/liburing/test/fdinfo.c

@@ -57,7 +57,7 @@ static void fdinfo_read(struct io_uring *ring)
 
 static int __test_io(const char *file, struct io_uring *ring, int write,
 		     int buffered, int sqthread, int fixed, int nonvec,
-		     int buf_select, int seq, int exp_len)
+		     int buf_select, int seq, int exp_len, int mixed)
 {
 	struct io_uring_sqe *sqe;
 	struct io_uring_cqe *cqe;
@@ -66,9 +66,9 @@ static int __test_io(const char *file, struct io_uring *ring, int write,
 	off_t offset;
 
 #ifdef VERBOSE
-	fprintf(stdout, "%s: start %d/%d/%d/%d/%d: ", __FUNCTION__, write,
+	fprintf(stdout, "%s: start %d/%d/%d/%d/%d/%d: ", __FUNCTION__, write,
 							buffered, sqthread,
-							fixed, nonvec);
+							fixed, nonvec, mixed);
 #endif
 	if (write)
 		open_flags = O_WRONLY;
@@ -107,6 +107,17 @@ static int __test_io(const char *file, struct io_uring *ring, int write,
 
 	offset = 0;
 	for (i = 0; i < BUFFERS; i++) {
+		if (mixed && i & 1) {
+			sqe = io_uring_get_sqe128(ring);
+			if (!sqe) {
+				fprintf(stderr, "sqe get failed\n");
+				goto err;
+			}
+			io_uring_prep_nop128(sqe);
+			sqe->user_data = i;
+			continue;
+		}
+
 		sqe = io_uring_get_sqe(ring);
 		if (!sqe) {
 			fprintf(stderr, "sqe get failed\n");
@@ -167,12 +178,17 @@ static int __test_io(const char *file, struct io_uring *ring, int write,
 			offset += BS;
 	}
 
+	__io_uring_flush_sq(ring);
 	fdinfo_read(ring);
 
 	ret = io_uring_submit(ring);
-	if (ret != BUFFERS) {
+	if (!mixed && ret != BUFFERS) {
 		fprintf(stderr, "submit got %d, wanted %d\n", ret, BUFFERS);
 		goto err;
+	} else if (mixed && ret != BUFFERS + (BUFFERS >> 1)) {
+		fprintf(stderr, "submit got %d, wanted %d\n", ret,
+			BUFFERS + (BUFFERS >> 1));
+		goto err;
 	}
 
 	for (i = 0; i < 10; i++) {
@@ -186,7 +202,13 @@ static int __test_io(const char *file, struct io_uring *ring, int write,
 			fprintf(stderr, "wait_cqe=%d\n", ret);
 			goto err;
 		}
-		if (cqe->res == -EINVAL && nonvec) {
+		if (mixed && cqe->user_data & 1) {
+			if (cqe->user_data > 32) {
+				fprintf(stderr, "cqe user data %lld, max expected %d\n",
+					cqe->user_data, BUFFERS - 1);
+				goto err;
+			}
+		} else if (cqe->res == -EINVAL && nonvec) {
 			if (!warned) {
 				fprintf(stdout, "Non-vectored IO not "
 					"supported, skipping\n");
@@ -252,15 +274,19 @@ err:
 	return 1;
 }
 static int test_io(const char *file, int write, int buffered, int sqthread,
-		   int fixed, int nonvec, int exp_len)
+		   int fixed, int nonvec, int exp_len, int mixed)
 {
 	struct io_uring ring;
-	int ret, ring_flags = 0;
+	int ret, ring_flags = 0, entries = 64;
 
 	if (sqthread)
 		ring_flags = IORING_SETUP_SQPOLL;
+	if (mixed) {
+		ring_flags |= IORING_SETUP_SQE_MIXED;
+		entries = 128;
+	}
 
-	ret = t_create_ring(64, &ring, ring_flags);
+	ret = t_create_ring(entries, &ring, ring_flags);
 	if (ret == T_SETUP_SKIP)
 		return 0;
 	if (ret != T_SETUP_OK) {
@@ -269,7 +295,7 @@ static int test_io(const char *file, int write, int buffered, int sqthread,
 	}
 
 	ret = __test_io(file, &ring, write, buffered, sqthread, fixed, nonvec,
-			0, 0, exp_len);
+			0, 0, exp_len, mixed);
 	io_uring_queue_exit(&ring);
 	return ret;
 }
@@ -379,17 +405,18 @@ int main(int argc, char *argv[])
 	vecs = t_create_buffers(BUFFERS, BS);
 
 	/* if we don't have nonvec read, skip testing that */
-	nr = has_nonvec_read() ? 32 : 16;
+	nr = has_nonvec_read() ? 64 : 32;
 
 	for (i = 0; i < nr; i++) {
 		int write = (i & 1) != 0;
 		int buffered = (i & 2) != 0;
 		int sqthread = (i & 4) != 0;
 		int fixed = (i & 8) != 0;
-		int nonvec = (i & 16) != 0;
+		int mixed = (i & 16) != 0;
+		int nonvec = (i & 32) != 0;
 
 		ret = test_io(fname, write, buffered, sqthread, fixed, nonvec,
-			      BS);
+			      BS, mixed);
 		if (ret == T_EXIT_SKIP)
 			continue;
 		if (ret) {

data/vendor/liburing/test/fifo-futex-poll.c

@@ -0,0 +1,114 @@
+/* SPDX-License-Identifier: MIT */
+/*
+ * Description: Run a failing futex wait command followed up a POLL_ADD
+ *		on a fifo.
+ *
+ */
+#include <stdio.h>
+#include <stdlib.h>
+#include <linux/futex.h>
+#include <linux/poll.h>
+#include <unistd.h>
+
+#include "liburing.h"
+#include "helpers.h"
+
+#ifndef FUTEX2_SIZE_U32
+#define FUTEX2_SIZE_U32		0x02
+#endif
+
+int main(int argc, char *argv[])
+{
+	struct io_uring_sqe *sqe;
+	struct io_uring_cqe *cqe;
+	struct io_uring ring;
+	int ret, pipe;
+
+	if (argc > 1)
+		return T_EXIT_SKIP;
+
+	/*
+	 * Setup a FIFO for polling. FIFOs are interesting because they end
+	 * up using two wait queue entries for poll, hitting double poll
+	 * in io_uring.
+	 */
+	ret = mkfifo("fifo", 0644);
+	if (ret < 0) {
+		perror("mkfifo");
+		return 1;
+	}
+	pipe = open("fifo", O_RDWR);
+	if (pipe < 0) {
+		perror("open fifo");
+		ret = T_EXIT_FAIL;
+		goto err;
+	}
+
+	ret = io_uring_queue_init(64, &ring, 0);
+	if (ret) {
+		ret = T_EXIT_FAIL;
+		goto err;
+	}
+
+	/*
+	 * Submit invalid futex wait, will error inline.
+	 */
+	sqe = io_uring_get_sqe(&ring);
+	io_uring_prep_futex_wait(sqe, NULL, 0, FUTEX_BITSET_MATCH_ANY, FUTEX2_SIZE_U32, 0);
+	sqe->user_data = 1;
+	io_uring_submit(&ring);
+
+	ret = io_uring_wait_cqe(&ring, &cqe);
+	if (ret) {
+		fprintf(stderr, "wait %d \n", ret);
+		ret = T_EXIT_FAIL;
+		goto err_ring;
+	}
+	if (cqe->res != -EFAULT) {
+		if (cqe->res == -EINVAL) {
+			ret = T_EXIT_SKIP;
+			goto err_ring;
+		}
+		fprintf(stderr, "Unexpected futex return: %d\n", cqe->res);
+		ret = T_EXIT_FAIL;
+		goto err_ring;
+	}
+	io_uring_cqe_seen(&ring, cqe);
+
+	ret = io_uring_register_files(&ring, &pipe, 1);
+	if (ret < 0) {
+		fprintf(stderr, "io_uring register files failed \n");
+		ret = T_EXIT_FAIL;
+		goto err_ring;
+	}
+
+	/*
+	 * Submit poll request for POLLIN on the fifo
+	 */
+	sqe = io_uring_get_sqe(&ring);
+	io_uring_prep_poll_add(sqe, 0, POLLIN);
+	sqe->flags |= IOSQE_FIXED_FILE;
+	sqe->user_data = 2;
+	io_uring_submit(&ring);
+
+	/*
+	 * Trigger the poll request
+	 */
+	ret = write(pipe, "FIFO test\n", 10);
+	if (ret < 0)
+		perror("fifo write");
+
+	ret = io_uring_wait_cqe(&ring, &cqe);
+	if (ret) {
+		fprintf(stderr, "wait %d \n", ret);
+		ret = T_EXIT_FAIL;
+	}
+	io_uring_cqe_seen(&ring, cqe);
+err_ring:
+	io_uring_queue_exit(&ring);
+err:
+	if (pipe != -1)
+		close(pipe);
+	unlink("fifo");
+	return ret;
+}

data/vendor/liburing/test/fifo-nonblock-read.c

@@ -18,7 +18,6 @@ int main(int argc, char *argv[])
 	struct io_uring ring;
 	char buf[32];
 	int fds[2];
-	int flags;
 	int ret;
 
 	io_uring_queue_init(1, &ring, 0);
@@ -28,17 +27,7 @@ int main(int argc, char *argv[])
 		return T_EXIT_FAIL;
 	}
 
-	flags = fcntl(fds[0], F_GETFL, 0);
-	if (flags < 0) {
-		perror("fcntl get");
-		return T_EXIT_FAIL;
-	}
-	flags |= O_NONBLOCK;
-	ret = fcntl(fds[0], F_SETFL, flags);
-	if (ret < 0) {
-		perror("fcntl set");
-		return T_EXIT_FAIL;
-	}
+	t_set_nonblock(fds[0]);
 
 	sqe = io_uring_get_sqe(&ring);
 	io_uring_prep_read(sqe, fds[0], buf, sizeof(buf), 0);

data/vendor/liburing/test/futex.c

@@ -149,7 +149,7 @@ static int test(int flags, int vectored)
 		return ret;
 
 	for (i = 0; i < LOOPS; i++) {
-		int async_cancel = (!i % 2);
+		int async_cancel = !(i % 2);
 		int async_wait = !(i % 3);
 		ret = __test(&ring, vectored, async_wait, async_cancel);
 		if (ret) {

data/vendor/liburing/test/helpers.c

@@ -136,9 +136,10 @@ enum t_setup_ret t_create_ring_params(int depth, struct io_uring *ring,
 		fprintf(stdout, "SQPOLL skipped for regular user\n");
 		return T_SETUP_SKIP;
 	}
+	if (ret == -EINVAL)
+		return T_SETUP_SKIP;
 
-	if (ret != -EINVAL)
-		fprintf(stderr, "queue_init: %s\n", strerror(-ret));
+	fprintf(stderr, "queue_init: %s\n", strerror(-ret));
 	return ret;
 }
 
@@ -484,3 +485,99 @@ int t_create_socketpair_ip(struct sockaddr_storage *addr,
 	}
 	return 0;
 }
+
+static void __t_toggle_nonblock(int fd, int set)
+{
+	int flags;
+
+	flags = fcntl(fd, F_GETFL, 0);
+	if (flags < 0)
+		t_error(1, errno, "fcntl F_GETFL");
+	if (set)
+		flags |= O_NONBLOCK;
+	else
+		flags &= ~O_NONBLOCK;
+	if (fcntl(fd, F_SETFL, flags) < 0)
+		t_error(1, errno, "fcntl F_SETFL");
+}
+
+void t_set_nonblock(int fd)
+{
+	__t_toggle_nonblock(fd, 1);
+}
+
+void t_clear_nonblock(int fd)
+{
+	__t_toggle_nonblock(fd, 0);
+}
+
+int t_submit_and_wait_single(struct io_uring *ring, struct io_uring_cqe **cqe)
+{
+	int ret;
+
+	ret = io_uring_submit(ring);
+	if (ret <= 0) {
+		fprintf(stderr, "sqe submit failed: %d\n", ret);
+		return -1;
+	}
+	ret = io_uring_wait_cqe(ring, cqe);
+	if (ret < 0) {
+		fprintf(stderr, "wait completion %d\n", ret);
+		return ret;
+	}
+	return 0;
+}
+
+size_t t_iovec_data_length(struct iovec *iov, unsigned iov_len)
+{
+	size_t sz = 0;
+	int i;
+
+	for (i = 0; i < iov_len; i++)
+		sz += iov[i].iov_len;
+	return sz;
+}
+
+#define t_min(a, b) ((a) < (b) ? (a) : (b))
+
+unsigned long t_compare_data_iovec(struct iovec *iov_src, unsigned nr_src,
+				   struct iovec *iov_dst, unsigned nr_dst)
+{
+	size_t src_len = t_iovec_data_length(iov_src, nr_src);
+	size_t dst_len = t_iovec_data_length(iov_dst, nr_dst);
+	size_t len_left = t_min(src_len, dst_len);
+	unsigned long src_off = 0, dst_off = 0;
+	unsigned long offset = 0;
+
+	while (offset != len_left) {
+		size_t len = len_left - offset;
+		unsigned long i;
+
+		len = t_min(len, iov_src->iov_len - src_off);
+		len = t_min(len, iov_dst->iov_len - dst_off);
+
+		for (i = 0; i < len; i++) {
+			char csrc = ((char *)iov_src->iov_base)[src_off + i];
+			char cdst = ((char *)iov_dst->iov_base)[dst_off + i];
+
+			if (csrc != cdst) {
+				fprintf(stderr, "data mismatch, %i vs %i\n",
+					csrc, cdst);
+				return -EINVAL;
+			}
+		}
+
+		src_off += len;
+		dst_off += len;
+		if (src_off == iov_src->iov_len) {
+			src_off = 0;
+			iov_src++;
+		}
+		if (dst_off == iov_dst->iov_len) {
+			dst_off = 0;
+			iov_dst++;
+		}
+		offset += len;
+	}
+	return 0;
+}

data/vendor/liburing/test/helpers.h

@@ -99,6 +99,8 @@ enum t_setup_ret t_register_buffers(struct io_uring *ring,
 				    unsigned nr_iovecs);
 
 bool t_probe_defer_taskrun(void);
+void t_set_nonblock(int fd);
+void t_clear_nonblock(int fd);
 
 unsigned __io_uring_flush_sq(struct io_uring *ring);
 
@@ -120,6 +122,13 @@ unsigned long long mtime_since_now(struct timeval *tv);
 unsigned long long utime_since(const struct timeval *s, const struct timeval *e);
 unsigned long long utime_since_now(struct timeval *tv);
 
+int t_submit_and_wait_single(struct io_uring *ring, struct io_uring_cqe **cqe);
+
+size_t t_iovec_data_length(struct iovec *iov, unsigned iov_len);
+
+unsigned long t_compare_data_iovec(struct iovec *iov_src, unsigned nr_src,
+				   struct iovec *iov_dst, unsigned nr_dst);
+
 #ifdef __cplusplus
 }
 #endif

data/vendor/liburing/test/io_uring_passthrough.c

@@ -141,18 +141,16 @@ static int __test_io(const char *file, struct io_uring *ring, int tc, int read,
 
 		if (sqthread)
 			use_fd = 0;
+		if (nonvec)
+			io_uring_prep_uring_cmd(sqe, NVME_URING_CMD_IO, use_fd);
+		else
+			io_uring_prep_uring_cmd(sqe, NVME_URING_CMD_IO_VEC, use_fd);
 		if (fixed && (i & 1))
 			do_fixed = 0;
 		if (do_fixed)
 			sqe->buf_index = 0;
 		if (async)
 			sqe->flags |= IOSQE_ASYNC;
-		if (nonvec)
-			sqe->cmd_op = NVME_URING_CMD_IO;
-		else
-			sqe->cmd_op = NVME_URING_CMD_IO_VEC;
-		sqe->fd = use_fd;
-		sqe->opcode = IORING_OP_URING_CMD;
 		if (do_fixed)
 			sqe->uring_cmd_flags |= IORING_URING_CMD_FIXED;
 		sqe->user_data = ((uint64_t)offset << 32) | i;
@@ -328,9 +326,7 @@ static int test_invalid_passthru_submit(const char *file)
 	}
 
 	sqe = io_uring_get_sqe(&ring);
-	io_uring_prep_read(sqe, fd, vecs[0].iov_base, vecs[0].iov_len, 0);
-	sqe->cmd_op = NVME_URING_CMD_IO;
-	sqe->opcode = IORING_OP_URING_CMD;
+	io_uring_prep_uring_cmd(sqe, NVME_URING_CMD_IO, fd);
 	sqe->user_data = 1;
 	cmd = (struct nvme_uring_cmd *)sqe->cmd;
 	memset(cmd, 0, sizeof(struct nvme_uring_cmd));
@@ -401,10 +397,8 @@ static int test_io_uring_submit_enters(const char *file)
 		__u32 nlb;
 
 		sqe = io_uring_get_sqe(&ring);
-		io_uring_prep_readv(sqe, fd, &vecs[i], 1, offset);
+		io_uring_prep_uring_cmd(sqe, NVME_URING_CMD_IO, fd);
 		sqe->user_data = i;
-		sqe->opcode = IORING_OP_URING_CMD;
-		sqe->cmd_op = NVME_URING_CMD_IO;
 		cmd = (struct nvme_uring_cmd *)sqe->cmd;
 		memset(cmd, 0, sizeof(struct nvme_uring_cmd));