uringmachine 0.19 → 0.20.0

data/vendor/liburing/test/fifo-nonblock-read.c

@@ -18,7 +18,6 @@ int main(int argc, char *argv[])
 	struct io_uring ring;
 	char buf[32];
 	int fds[2];
-	int flags;
 	int ret;
 
 	io_uring_queue_init(1, &ring, 0);
@@ -28,17 +27,7 @@ int main(int argc, char *argv[])
 		return T_EXIT_FAIL;
 	}
 
-	flags = fcntl(fds[0], F_GETFL, 0);
-	if (flags < 0) {
-		perror("fcntl get");
-		return T_EXIT_FAIL;
-	}
-	flags |= O_NONBLOCK;
-	ret = fcntl(fds[0], F_SETFL, flags);
-	if (ret < 0) {
-		perror("fcntl set");
-		return T_EXIT_FAIL;
-	}
+	t_set_nonblock(fds[0]);
 
 	sqe = io_uring_get_sqe(&ring);
 	io_uring_prep_read(sqe, fds[0], buf, sizeof(buf), 0);

data/vendor/liburing/test/futex.c

@@ -149,7 +149,7 @@ static int test(int flags, int vectored)
 		return ret;
 
 	for (i = 0; i < LOOPS; i++) {
-		int async_cancel = (!i % 2);
+		int async_cancel = !(i % 2);
 		int async_wait = !(i % 3);
 		ret = __test(&ring, vectored, async_wait, async_cancel);
 		if (ret) {

data/vendor/liburing/test/helpers.c

@@ -136,9 +136,10 @@ enum t_setup_ret t_create_ring_params(int depth, struct io_uring *ring,
 		fprintf(stdout, "SQPOLL skipped for regular user\n");
 		return T_SETUP_SKIP;
 	}
+	if (ret == -EINVAL)
+		return T_SETUP_SKIP;
 
-	if (ret != -EINVAL)
-		fprintf(stderr, "queue_init: %s\n", strerror(-ret));
+	fprintf(stderr, "queue_init: %s\n", strerror(-ret));
 	return ret;
 }
 
@@ -484,3 +485,99 @@ int t_create_socketpair_ip(struct sockaddr_storage *addr,
 	}
 	return 0;
 }
+
+static void __t_toggle_nonblock(int fd, int set)
+{
+	int flags;
+
+	flags = fcntl(fd, F_GETFL, 0);
+	if (flags < 0)
+		t_error(1, errno, "fcntl F_GETFL");
+	if (set)
+		flags |= O_NONBLOCK;
+	else
+		flags &= ~O_NONBLOCK;
+	if (fcntl(fd, F_SETFL, flags) < 0)
+		t_error(1, errno, "fcntl F_SETFL");
+}
+
+void t_set_nonblock(int fd)
+{
+	__t_toggle_nonblock(fd, 1);
+}
+
+void t_clear_nonblock(int fd)
+{
+	__t_toggle_nonblock(fd, 0);
+}
+
+int t_submit_and_wait_single(struct io_uring *ring, struct io_uring_cqe **cqe)
+{
+	int ret;
+
+	ret = io_uring_submit(ring);
+	if (ret <= 0) {
+		fprintf(stderr, "sqe submit failed: %d\n", ret);
+		return -1;
+	}
+	ret = io_uring_wait_cqe(ring, cqe);
+	if (ret < 0) {
+		fprintf(stderr, "wait completion %d\n", ret);
+		return ret;
+	}
+	return 0;
+}
+
+size_t t_iovec_data_length(struct iovec *iov, unsigned iov_len)
+{
+	size_t sz = 0;
+	int i;
+
+	for (i = 0; i < iov_len; i++)
+		sz += iov[i].iov_len;
+	return sz;
+}
+
+#define t_min(a, b) ((a) < (b) ? (a) : (b))
+
+unsigned long t_compare_data_iovec(struct iovec *iov_src, unsigned nr_src,
+				   struct iovec *iov_dst, unsigned nr_dst)
+{
+	size_t src_len = t_iovec_data_length(iov_src, nr_src);
+	size_t dst_len = t_iovec_data_length(iov_dst, nr_dst);
+	size_t len_left = t_min(src_len, dst_len);
+	unsigned long src_off = 0, dst_off = 0;
+	unsigned long offset = 0;
+
+	while (offset != len_left) {
+		size_t len = len_left - offset;
+		unsigned long i;
+
+		len = t_min(len, iov_src->iov_len - src_off);
+		len = t_min(len, iov_dst->iov_len - dst_off);
+
+		for (i = 0; i < len; i++) {
+			char csrc = ((char *)iov_src->iov_base)[src_off + i];
+			char cdst = ((char *)iov_dst->iov_base)[dst_off + i];
+
+			if (csrc != cdst) {
+				fprintf(stderr, "data mismatch, %i vs %i\n",
+					csrc, cdst);
+				return -EINVAL;
+			}
+		}
+
+		src_off += len;
+		dst_off += len;
+		if (src_off == iov_src->iov_len) {
+			src_off = 0;
+			iov_src++;
+		}
+		if (dst_off == iov_dst->iov_len) {
+			dst_off = 0;
+			iov_dst++;
+		}
+		offset += len;
+	}
+	return 0;
+}

data/vendor/liburing/test/helpers.h

@@ -99,6 +99,8 @@ enum t_setup_ret t_register_buffers(struct io_uring *ring,
 				    unsigned nr_iovecs);
 
 bool t_probe_defer_taskrun(void);
+void t_set_nonblock(int fd);
+void t_clear_nonblock(int fd);
 
 unsigned __io_uring_flush_sq(struct io_uring *ring);
 
@@ -120,6 +122,13 @@ unsigned long long mtime_since_now(struct timeval *tv);
 unsigned long long utime_since(const struct timeval *s, const struct timeval *e);
 unsigned long long utime_since_now(struct timeval *tv);
 
+int t_submit_and_wait_single(struct io_uring *ring, struct io_uring_cqe **cqe);
+
+size_t t_iovec_data_length(struct iovec *iov, unsigned iov_len);
+
+unsigned long t_compare_data_iovec(struct iovec *iov_src, unsigned nr_src,
+				   struct iovec *iov_dst, unsigned nr_dst);
+
 #ifdef __cplusplus
 }
 #endif

data/vendor/liburing/test/io_uring_passthrough.c

@@ -141,18 +141,16 @@ static int __test_io(const char *file, struct io_uring *ring, int tc, int read,
 
 		if (sqthread)
 			use_fd = 0;
+		if (nonvec)
+			io_uring_prep_uring_cmd(sqe, NVME_URING_CMD_IO, use_fd);
+		else
+			io_uring_prep_uring_cmd(sqe, NVME_URING_CMD_IO_VEC, use_fd);
 		if (fixed && (i & 1))
 			do_fixed = 0;
 		if (do_fixed)
 			sqe->buf_index = 0;
 		if (async)
 			sqe->flags |= IOSQE_ASYNC;
-		if (nonvec)
-			sqe->cmd_op = NVME_URING_CMD_IO;
-		else
-			sqe->cmd_op = NVME_URING_CMD_IO_VEC;
-		sqe->fd = use_fd;
-		sqe->opcode = IORING_OP_URING_CMD;
 		if (do_fixed)
 			sqe->uring_cmd_flags |= IORING_URING_CMD_FIXED;
 		sqe->user_data = ((uint64_t)offset << 32) | i;
@@ -328,9 +326,7 @@ static int test_invalid_passthru_submit(const char *file)
 	}
 
 	sqe = io_uring_get_sqe(&ring);
-	io_uring_prep_read(sqe, fd, vecs[0].iov_base, vecs[0].iov_len, 0);
-	sqe->cmd_op = NVME_URING_CMD_IO;
-	sqe->opcode = IORING_OP_URING_CMD;
+	io_uring_prep_uring_cmd(sqe, NVME_URING_CMD_IO, fd);
 	sqe->user_data = 1;
 	cmd = (struct nvme_uring_cmd *)sqe->cmd;
 	memset(cmd, 0, sizeof(struct nvme_uring_cmd));
@@ -401,10 +397,8 @@ static int test_io_uring_submit_enters(const char *file)
 		__u32 nlb;
 
 		sqe = io_uring_get_sqe(&ring);
-		io_uring_prep_readv(sqe, fd, &vecs[i], 1, offset);
+		io_uring_prep_uring_cmd(sqe, NVME_URING_CMD_IO, fd);
 		sqe->user_data = i;
-		sqe->opcode = IORING_OP_URING_CMD;
-		sqe->cmd_op = NVME_URING_CMD_IO;
 		cmd = (struct nvme_uring_cmd *)sqe->cmd;
 		memset(cmd, 0, sizeof(struct nvme_uring_cmd));
 

data/vendor/liburing/test/mock_file.c

@@ -0,0 +1,379 @@
+#include <errno.h>
+#include <stdio.h>
+#include <unistd.h>
+#include <stdlib.h>
+#include <string.h>
+#include <fcntl.h>
+#include <poll.h>
+#include <assert.h>
+
+#include "liburing.h"
+#include "test.h"
+#include "helpers.h"
+
+#include "mock_file.h"
+
+static struct io_uring mgr_ring;
+static __u64 mock_features;
+static int mgr_fd;
+
+static bool has_feature(int feature)
+{
+	return mock_features >= feature;
+}
+
+static int setup_mgr(void)
+{
+	struct io_uring_mock_probe mp;
+	struct io_uring_cqe *cqe;
+	struct io_uring_sqe *sqe;
+	int ret;
+
+	ret = mgr_fd = open("/dev/io_uring_mock", O_RDWR);
+	if (mgr_fd < 0) {
+		if (errno == EACCES)
+			printf("No permission to open /dev/io_uring_mock, skipping\n");
+		else
+			printf("no io_uring mock files, skip\n");
+		return T_EXIT_SKIP;
+	}
+
+	ret = io_uring_queue_init(8, &mgr_ring, 0);
+	if (ret) {
+		fprintf(stderr, "mgr ring setup failed %i\n", ret);
+		return T_EXIT_FAIL;
+	}
+
+	memset(&mp, 0, sizeof(mp));
+	sqe = io_uring_get_sqe(&mgr_ring);
+	io_uring_prep_uring_cmd(sqe, IORING_MOCK_MGR_CMD_PROBE, mgr_fd);
+	sqe->addr  = (__u64)(unsigned long)&mp;
+	sqe->len = sizeof(mp);
+
+	ret = t_submit_and_wait_single(&mgr_ring, &cqe);
+	if (ret || cqe->res) {
+		fprintf(stderr, "probe cmd failed %i %i\n", ret, cqe->res);
+		return T_EXIT_FAIL;
+	}
+
+	io_uring_cqe_seen(&mgr_ring, cqe);
+	mock_features = mp.features;
+	return 0;
+}
+
+static int create_mock_file(struct io_uring_mock_create *mc)
+{
+	struct io_uring_cqe *cqe;
+	struct io_uring_sqe *sqe;
+	int ret;
+
+	sqe = io_uring_get_sqe(&mgr_ring);
+	io_uring_prep_uring_cmd(sqe, IORING_MOCK_MGR_CMD_CREATE, mgr_fd);
+	sqe->addr  = (__u64)(unsigned long)mc;
+	sqe->len = sizeof(*mc);
+
+	ret = t_submit_and_wait_single(&mgr_ring, &cqe);
+	if (ret || cqe->res) {
+		fprintf(stderr, "file create cmd failed %i %i\n", ret, cqe->res);
+		return T_EXIT_FAIL;
+	}
+	io_uring_cqe_seen(&mgr_ring, cqe);
+	return 0;
+}
+
+static int t_copy_regvec(struct io_uring *ring, int mock_fd,
+			 struct iovec *iov, unsigned iov_len, char *buf,
+			 bool from_iov)
+{
+	struct io_uring_cqe *cqe;
+	struct io_uring_sqe *sqe;
+	int ret;
+
+	sqe = io_uring_get_sqe(ring);
+	io_uring_prep_uring_cmd(sqe, IORING_MOCK_CMD_COPY_REGBUF, mock_fd);
+	sqe->addr3 = (__u64)(unsigned long)buf;
+	sqe->addr = (__u64)(unsigned long)iov;
+	sqe->len = iov_len;
+	if (from_iov)
+		sqe->file_index = IORING_MOCK_COPY_FROM;
+	sqe->buf_index = from_iov ? 0 : 1;
+	sqe->user_data = 43;
+	sqe->uring_cmd_flags |= IORING_URING_CMD_FIXED;
+
+	ret = t_submit_and_wait_single(ring, &cqe);
+	if (ret)
+		t_error(1, ret, "submit/wait failed");
+
+	ret = cqe->res;
+	io_uring_cqe_seen(ring, cqe);
+	return ret;
+}
+
+static int t_copy_verify_regvec(struct io_uring *ring, int mock_fd,
+				struct iovec *iov, unsigned iov_len, char *buf,
+				bool from_iov)
+{
+	struct iovec iov2;
+	int ret;
+
+	ret = t_copy_regvec(ring, mock_fd, iov, iov_len, buf, from_iov);
+	if (ret < 0 || ret != t_iovec_data_length(iov, iov_len))
+		return ret < 0 ? ret : -1;
+
+	iov2.iov_base = buf;
+	iov2.iov_len = -1U;
+
+	ret = t_compare_data_iovec(iov, iov_len, &iov2, 1);
+	if (ret) {
+		fprintf(stderr, "iovec1 data mismatch %i\n", ret);
+		return -1;
+	}
+	return 0;
+}
+
+static int test_regvec_cmd(struct io_uring *ring, int mock_fd)
+{
+	struct iovec buf_iovec[2];
+	struct iovec iov[8];
+	size_t size = 4096 * 32;
+	char *buf_src, *buf_dst;
+	int i, ret;
+
+	buf_src = aligned_alloc(4096, size);
+	buf_dst = aligned_alloc(4096, size);
+	if (!buf_src || !buf_dst)
+		t_error(0, -ENOMEM, "can't allocate buffers");
+
+	for (i = 0; i < size; i++)
+		buf_src[i] = 'a' + (i % 26);
+
+	buf_iovec[0].iov_base = buf_src;
+	buf_iovec[0].iov_len = size;
+	buf_iovec[1].iov_base = buf_dst;
+	buf_iovec[1].iov_len = size;
+	ret = t_register_buffers(ring, buf_iovec, 2);
+	if (ret) {
+		free(buf_src);
+		free(buf_dst);
+		return ret == T_SETUP_SKIP ? 0 : T_EXIT_FAIL;
+	}
+
+	memset(buf_dst, 0, size);
+	iov[0].iov_len = size;
+	iov[0].iov_base = buf_src;
+	ret = t_copy_verify_regvec(ring, mock_fd, iov, 1, buf_dst, true);
+	if (ret < 0) {
+		fprintf(stderr, "t_copy_verify_regvec iovec1 failed %i\n", ret);
+		return T_EXIT_FAIL;
+	}
+
+	memset(buf_dst, 0, size);
+	iov[0].iov_len = size;
+	iov[0].iov_base = buf_dst;
+	ret = t_copy_verify_regvec(ring, mock_fd, iov, 1, buf_src, false);
+	if (ret < 0) {
+		fprintf(stderr, "t_copy_verify_regvec iovec1 reverse failed %i\n", ret);
+		return T_EXIT_FAIL;
+	}
+
+	memset(buf_dst, 0, size);
+	iov[0].iov_base = buf_src;
+	iov[0].iov_len = 5;
+	iov[1].iov_base = buf_src + 5;
+	iov[1].iov_len = 11;
+	iov[2].iov_base = buf_src + (4096 - 127);
+	iov[2].iov_len = 127;
+	iov[3].iov_base = buf_src + (4096 - 127);
+	iov[3].iov_len = 127 + 4096 + 13;
+	iov[4].iov_base = buf_src + 4 * 4096;
+	iov[4].iov_len = 4096 + 73;
+	iov[5].iov_base = buf_src + 7 * 4096 + 127;
+	iov[5].iov_len = 4096 * 11 + 132;
+	assert(t_iovec_data_length(iov, 6) <= size);
+	ret = t_copy_verify_regvec(ring, mock_fd, iov, 6, buf_dst, true);
+	if (ret < 0) {
+		fprintf(stderr, "t_copy_verify_regvec iovec6 failed %i\n", ret);
+		return T_EXIT_FAIL;
+	}
+
+	memset(buf_dst, 0, size);
+	iov[0].iov_base = buf_dst;
+	iov[0].iov_len = 5;
+	iov[1].iov_base = buf_dst + 5;
+	iov[1].iov_len = 11;
+	iov[2].iov_base = buf_dst + (4096 - 127);
+	iov[2].iov_len = 127;
+	iov[3].iov_base = buf_dst + 4 * 4096;
+	iov[3].iov_len = 4096 + 73;
+	iov[4].iov_base = buf_dst + 7 * 4096 + 127;
+	iov[4].iov_len = 4096 * 11 + 132;
+	assert(t_iovec_data_length(iov, 5) <= size);
+	ret = t_copy_verify_regvec(ring, mock_fd, iov, 5, buf_src, false);
+	if (ret < 0) {
+		fprintf(stderr, "t_copy_verify_regvec iovec6 reverse failed %i\n", ret);
+		return T_EXIT_FAIL;
+	}
+
+	free(buf_src);
+	free(buf_dst);
+	return 0;
+}
+
+static int test_cmds(void)
+{
+	struct io_uring_mock_create mc;
+	struct io_uring ring;
+	int ret, mock_fd;
+
+	memset(&mc, 0, sizeof(mc));
+	if (create_mock_file(&mc))
+		return T_EXIT_FAIL;
+	mock_fd = mc.out_fd;
+
+	ret = io_uring_queue_init(8, &ring, 0);
+	if (ret) {
+		fprintf(stderr, "ring setup failed: %d\n", ret);
+		return 1;
+	}
+
+	if (has_feature(IORING_MOCK_FEAT_CMD_COPY)) {
+		ret = test_regvec_cmd(&ring, mock_fd);
+		if (ret) {
+			fprintf(stderr, "test_regvec_cmd() failed\n");
+			return T_EXIT_FAIL;
+		}
+	} else {
+		printf("skip test_regvec_cmd()\n");
+	}
+
+	io_uring_queue_exit(&ring);
+	return 0;
+}
+
+static int test_reads(struct io_uring *ring, int mock_fd, void *buffer)
+{
+	struct io_uring_cqe *cqe;
+	struct io_uring_sqe *sqe;
+	int io_len = 4096;
+	int nr_reqs = 16;
+	int i, ret;
+
+	for (i = 0; i < nr_reqs; i++) {
+		sqe = io_uring_get_sqe(ring);
+		io_uring_prep_read(sqe, mock_fd, buffer, io_len, 0);
+		sqe->user_data = i;
+	}
+
+	ret = io_uring_submit(ring);
+	if (ret != nr_reqs) {
+		fprintf(stderr, "submit got %d, wanted %d\n", ret, nr_reqs);
+		return T_EXIT_FAIL;
+	}
+
+	for (i = 0; i < nr_reqs; i++) {
+		ret = io_uring_wait_cqe(ring, &cqe);
+		if (ret) {
+			fprintf(stderr, "wait_cqe=%d\n", ret);
+			return T_EXIT_FAIL;
+		}
+		if (cqe->res != io_len) {
+			fprintf(stderr, "unexpected cqe res %i, data %i\n",
+				cqe->res, (int)cqe->user_data);
+			return T_EXIT_FAIL;
+		}
+		io_uring_cqe_seen(ring, cqe);
+	}
+	return 0;
+}
+
+static int test_rw(void)
+{
+	void *buffer;
+	struct io_uring ring;
+	int ret, i;
+
+	if (!has_feature(IORING_MOCK_FEAT_RW_ZERO)) {
+		printf("no mock read-write support, skip\n");
+		return T_EXIT_SKIP;
+	}
+
+	buffer = malloc(4096);
+	if (!buffer) {
+		fprintf(stderr, "can't allocate buffers\n");
+		return T_EXIT_FAIL;
+	}
+
+	ret = io_uring_queue_init(32, &ring, 0);
+	if (ret) {
+		fprintf(stderr, "ring setup failed: %d\n", ret);
+		return 1;
+	}
+
+	for (i = 0; i < 8; i++) {
+		struct io_uring_mock_create mc;
+		bool nowait = i & 1;
+		bool async = i & 2;
+		bool poll = i & 4;
+		int mock_fd;
+
+		memset(&mc, 0, sizeof(mc));
+		if (poll) {
+			if (!has_feature(IORING_MOCK_FEAT_POLL))
+				continue;
+			mc.flags |= IORING_MOCK_CREATE_F_POLL;
+		}
+		if (nowait) {
+			if (!has_feature(IORING_MOCK_FEAT_RW_NOWAIT))
+				continue;
+			mc.flags |= IORING_MOCK_CREATE_F_SUPPORT_NOWAIT;
+		}
+		if (async) {
+			if (!has_feature(IORING_MOCK_FEAT_RW_ASYNC))
+				continue;
+			mc.rw_delay_ns = 1000 * 1000 * 50;
+		}
+		mc.file_size = 10 * (1UL << 20);
+		if (create_mock_file(&mc))
+			return T_EXIT_FAIL;
+		mock_fd = mc.out_fd;
+
+		ret = test_reads(&ring, mock_fd, buffer);
+		if (ret) {
+			fprintf(stderr, "rw failed %i/%i/%i\n",
+				nowait, async, poll);
+			return T_EXIT_FAIL;
+		}
+
+		close(mock_fd);
+	}
+
+	free(buffer);
+	io_uring_queue_exit(&ring);
+	return 0;
+}
+
+int main(int argc, char *argv[])
+{
+	int ret;
+
+	if (argc > 1)
+		return T_EXIT_SKIP;
+
+	ret = setup_mgr();
+	if (ret)
+		return ret;
+
+	ret = test_cmds();
+	if (ret)
+		return T_EXIT_FAIL;
+
+	ret = test_rw();
+	if (ret) {
+		fprintf(stderr, "test_rw failed %i\n", ret);
+		return T_EXIT_FAIL;
+	}
+
+	io_uring_queue_exit(&mgr_ring);
+	close(mgr_fd);
+	return 0;
+}

data/vendor/liburing/test/mock_file.h

@@ -0,0 +1,47 @@
+#ifndef LINUX_IO_URING_MOCK_FILE_H
+#define LINUX_IO_URING_MOCK_FILE_H
+
+#include <linux/types.h>
+
+enum {
+	IORING_MOCK_FEAT_CMD_COPY,
+	IORING_MOCK_FEAT_RW_ZERO,
+	IORING_MOCK_FEAT_RW_NOWAIT,
+	IORING_MOCK_FEAT_RW_ASYNC,
+	IORING_MOCK_FEAT_POLL,
+
+	IORING_MOCK_FEAT_END,
+};
+
+struct io_uring_mock_probe {
+	__u64		features;
+	__u64		__resv[9];
+};
+
+enum {
+	IORING_MOCK_CREATE_F_SUPPORT_NOWAIT			= 1,
+	IORING_MOCK_CREATE_F_POLL				= 2,
+};
+
+struct io_uring_mock_create {
+	__u32		out_fd;
+	__u32		flags;
+	__u64		file_size;
+	__u64		rw_delay_ns;
+	__u64		__resv[13];
+};
+
+enum {
+	IORING_MOCK_MGR_CMD_PROBE,
+	IORING_MOCK_MGR_CMD_CREATE,
+};
+
+enum {
+	IORING_MOCK_CMD_COPY_REGBUF,
+};
+
+enum {
+	IORING_MOCK_COPY_FROM			= 1,
+};
+
+#endif

data/vendor/liburing/test/nop.c

@@ -83,7 +83,7 @@ static int test_single_nop(struct io_uring *ring, unsigned req_flags)
 		goto err;
 	}
 	if (!cqe->user_data) {
-		fprintf(stderr, "Unexpected 0 user_data\n");
+		fprintf(stderr, "Unexpected 0 user_data: %ld\n", (long) cqe->user_data);
 		goto err;
 	}
 	if (cqe32) {
@@ -140,7 +140,7 @@ static int test_barrier_nop(struct io_uring *ring, unsigned req_flags)
 			goto err;
 		}
 		if (!cqe->user_data) {
-			fprintf(stderr, "Unexpected 0 user_data\n");
+			fprintf(stderr, "Unexpected 0 user_data: %ld\n", (long) cqe->user_data);
 			goto err;
 		}
 		if (cqe32) {