uringmachine 0.10 → 0.11

data/vendor/liburing/test/iowait.c

@@ -0,0 +1,141 @@
+/* SPDX-License-Identifier: MIT */
+/*
+ * Description: test toggling of iowait on kernels that support it
+ *
+ */
+#include <stdio.h>
+#include <stdlib.h>
+#include <unistd.h>
+#include <string.h>
+
+#include "liburing.h"
+#include "helpers.h"
+
+static int get_iowait(int cpu)
+{
+	char cpu_buf[32], this_cpu[32];
+	int user, nice, system, idle, iowait, ret;
+	FILE *file;
+
+	file = fopen("/proc/stat", "r");
+	if (!file) {
+		perror("fopen stat");
+		return 0;
+	}
+
+	sprintf(this_cpu, "cpu%d", cpu);
+	ret = 0;
+	do {
+		int r;
+		r = fscanf(file, "%s %d %d %d %d %d", cpu_buf, &user, &nice,
+				&system, &idle, &iowait);
+		if (r != 6)
+			continue;
+		if (strncmp(cpu_buf, this_cpu, strlen(this_cpu)))
+			continue;
+		ret = iowait;
+		break;
+	} while (1);
+
+	fclose(file);
+	return ret;
+}
+
+static int test(struct io_uring *ring, int with_iowait, int cpu)
+{
+	struct io_uring_sqe *sqe;
+	struct io_uring_cqe *cqe;
+	int iowait_pre, iowait_post;
+	struct __kernel_timespec ts;
+	int ret, fds[2], diff;
+	char buf[32];
+
+	if (!(ring->features & IORING_FEAT_NO_IOWAIT))
+		return T_EXIT_SKIP;
+
+	if (pipe(fds) < 0) {
+		perror("pipe");
+		return T_EXIT_FAIL;
+	}
+
+	ret = io_uring_set_iowait(ring, with_iowait);
+	if (ret) {
+		fprintf(stderr, "set_iowait=%d\n", ret);
+		return T_EXIT_FAIL;
+	}
+
+	sqe = io_uring_get_sqe(ring);
+	io_uring_prep_read(sqe, fds[0], buf, sizeof(buf), 0);
+	io_uring_submit(ring);
+
+	ts.tv_sec = 1;
+	ts.tv_nsec = 0;
+	iowait_pre = get_iowait(cpu);
+	ret = io_uring_wait_cqe_timeout(ring, &cqe, &ts);
+	if (ret != -ETIME) {
+		fprintf(stderr, "Unexpected wait ret: %d\n", ret);
+		return T_EXIT_FAIL;
+	}
+	iowait_post = get_iowait(cpu);
+	diff = iowait_post - iowait_pre;
+
+	close(fds[0]);
+	close(fds[1]);
+
+	ret = io_uring_wait_cqe(ring, &cqe);
+	if (ret) {
+		fprintf(stderr, "Unexpected wait ret: %d\n", ret);
+		return T_EXIT_FAIL;
+	}
+	io_uring_cqe_seen(ring, cqe);
+
+	if (with_iowait) {
+		if (diff < 90) {
+			fprintf(stderr, "iowait diff too small: %d\n", diff);
+			return T_EXIT_FAIL;
+		}
+	} else {
+		if (diff > 10) {
+			fprintf(stderr, "iowait diff too large: %d\n", diff);
+			return T_EXIT_FAIL;
+		}
+	}
+	return T_EXIT_PASS;
+}
+
+int main(int argc, char *argv[])
+{
+	struct io_uring ring;
+	cpu_set_t cpuset;
+	int ret;
+
+	CPU_ZERO(&cpuset);
+	CPU_SET(0, &cpuset);
+	if (sched_setaffinity(0, sizeof(cpuset), &cpuset))
+		return T_EXIT_SKIP;
+
+	ret = t_create_ring(64, &ring, 0);
+	if (ret == T_SETUP_SKIP)
+		return T_EXIT_SKIP;
+	else if (ret != T_SETUP_OK)
+		return T_EXIT_FAIL;
+
+	ret = test(&ring, 0, 0);
+	if (ret == T_EXIT_SKIP) {
+		return T_EXIT_SKIP;
+	} else if (ret == T_EXIT_FAIL) {
+		fprintf(stderr, "test 0 failed\n");
+		return T_EXIT_FAIL;
+	}
+
+	ret = test(&ring, 1, 0);
+	if (ret == T_EXIT_SKIP) {
+		return T_EXIT_SKIP;
+	} else if (ret == T_EXIT_FAIL) {
+		fprintf(stderr, "test 1 failed\n");
+		return T_EXIT_FAIL;
+	}
+
+	io_uring_queue_exit(&ring);
+	return 0;
+}

data/vendor/liburing/test/nvme.h

@@ -59,6 +59,7 @@ enum nvme_io_opcode {
 
 static int nsid;
 static __u32 lba_shift;
+static __u32 meta_size;
 
 struct nvme_lbaf {
 	__le16			ms;
@@ -157,6 +158,7 @@ static int nvme_get_info(const char *file)
 
 	lba_size = 1 << ns.lbaf[(ns.flbas & 0x0f)].ds;
 	lba_shift = ilog2(lba_size);
+	meta_size = ns.lbaf[(ns.flbas & 0x0f)].ms;
 
 	close(fd);
 	return 0;

data/vendor/liburing/test/pipe-bug.c

@@ -24,20 +24,26 @@ do {										\
 
 static int pipe_bug(void)
 {
-	struct io_uring_params p;
 	struct io_uring ring;
 	struct io_uring_sqe *sqe;
 	struct io_uring_cqe *cqe;
 	char buf[1024];
-	int fds[2];
+	int ret, fds[2];
 	struct __kernel_timespec to = {
 		.tv_sec = 1
 	};
 
-	CHECK(pipe(fds) == 0);
+	ret = io_uring_queue_init(8, &ring, 0);
+	/* can hit -ENOMEM due to repeated ring creation and teardowns */
+	if (ret == -ENOMEM) {
+		usleep(1000);
+		return 0;
+	} else if (ret) {
+		fprintf(stderr, "ring_init: %d\n", ret);
+		return 1;
+	}
 
-	memset(&p, 0, sizeof(p));
-	CHECK(t_create_ring_params(8, &ring, &p) == 0);
+	CHECK(pipe(fds) == 0);
 
 	/* WRITE */
 	sqe = io_uring_get_sqe(&ring);

data/vendor/liburing/test/pipe-eof.c

@@ -11,6 +11,7 @@
 #include <pthread.h>
 #include <string.h>
 #include "liburing.h"
+#include "helpers.h"
 
 #define BUFSIZE	512
 
@@ -41,13 +42,22 @@ int main(int argc, char *argv[])
 	struct data d;
 	int ret;
 
+	if (argc > 1)
+		return T_EXIT_SKIP;
+
 	if (pipe(d.fds) < 0) {
 		perror("pipe");
 		return 1;
 	}
 	d.str = buf;
 
-	io_uring_queue_init(8, &ring, 0);
+	ret = io_uring_queue_init(8, &ring, 0);
+	if (ret == -ENOMEM) {
+		return T_EXIT_SKIP;
+	} else if (ret) {
+		fprintf(stderr, "queue_init: %d\n", ret);
+		return T_EXIT_FAIL;
+	}
 
 	pthread_create(&thread, NULL, t, &d);
 

data/vendor/liburing/test/read-inc-file.c

@@ -0,0 +1,150 @@
+/* SPDX-License-Identifier: MIT */
+/*
+ * Description: test reading a normal file with incremental buffer
+ *		consumption. Some kernels had a bug where the initial part
+ *		of the buffer got skipped, test for that.
+ *
+ */
+#include <stdio.h>
+#include <unistd.h>
+#include <stdlib.h>
+#include <string.h>
+#include "liburing.h"
+#include "helpers.h"
+
+#define BUF_BGID	4
+#define BUF_BID		8
+
+static void arm_read(struct io_uring *ring, int fd, int offset)
+{
+	struct io_uring_sqe *sqe;
+
+	sqe = io_uring_get_sqe(ring);
+	io_uring_prep_read(sqe, fd, NULL, 80, offset);
+	sqe->flags = IOSQE_BUFFER_SELECT;
+	sqe->buf_group = BUF_BGID;
+	io_uring_submit(ring);
+}
+
+static int create_test_file(const char *fname)
+{
+	char buf[80], c;
+	int fd, i, ret;
+
+	fd = open(fname, O_WRONLY | O_CREAT | O_TRUNC, 0644);
+	if (fd < 0) {
+		perror("open");
+		return T_EXIT_FAIL;
+	}
+
+	c = 'a';
+	for (i = 0; i < 8; i++) {
+		memset(buf, c, sizeof(buf));
+		ret = write(fd, buf, sizeof(buf));
+		if (ret < 0) {
+			perror("write");
+			unlink(fname);
+			return T_EXIT_FAIL;
+		} else if (ret != sizeof(buf)) {
+			fprintf(stderr, "Short write: %d\n", ret);
+			unlink(fname);
+			return T_EXIT_FAIL;
+		}
+		c++;
+	}
+
+	close(fd);
+	return 0;
+}
+
+int main(int argc, char *argv[])
+{
+	struct io_uring_buf_ring *br;
+	struct io_uring_params p = { };
+	struct io_uring_cqe *cqe;
+	struct io_uring ring;
+	int tret, ret, fd, i;
+	char fname[64];
+	char c = 'a';
+	char *buf;
+	void *ptr;
+
+	if (argc > 1)
+		return T_EXIT_SKIP;
+
+	sprintf(fname, ".buf-inc-file.%d", getpid());
+	if (create_test_file(fname))
+		return T_EXIT_FAIL;
+
+	fd = open(fname, O_RDONLY);
+	if (fd < 0) {
+		perror("open");
+		goto err;
+	}
+
+	ret = io_uring_queue_init_params(64, &ring, &p);
+	if (ret) {
+		fprintf(stderr, "ring setup failed: %d\n", ret);
+		goto err;
+	}
+
+	if (posix_memalign((void **) &buf, 4096, 65536))
+		goto err;
+
+	tret = T_EXIT_SKIP;
+	br = io_uring_setup_buf_ring(&ring, 32, BUF_BGID, IOU_PBUF_RING_INC, &ret);
+	if (!br) {
+		if (ret == -EINVAL)
+			goto out;
+		fprintf(stderr, "Buffer ring register failed %d\n", ret);
+		goto err;
+	}
+
+	tret = T_EXIT_PASS;
+	io_uring_buf_ring_add(br, buf, 65536, BUF_BID, 31, 0);
+	io_uring_buf_ring_advance(br, 1);
+
+	memset(buf, 0, 65536);
+
+	ptr = buf;
+	for (i = 0; i < 4; i++) {
+		int bid;
+
+		arm_read(&ring, fd, i * 80);
+		ret = io_uring_wait_cqe(&ring, &cqe);
+		if (ret) {
+			fprintf(stderr, "wait %d\n", ret);
+			goto err;
+		}
+		if (!(cqe->flags & IORING_CQE_F_BUFFER)) {
+			fprintf(stderr, "buffer not assigned\n");
+			goto err;
+		}
+		bid = cqe->flags >> IORING_CQE_BUFFER_SHIFT;
+		if (bid != BUF_BID) {
+			fprintf(stderr, "got wrong buffer bid %d\n", bid);
+			goto err;
+		}
+		if (cqe->res != 80) {
+			fprintf(stderr, "bad read size %d\n", ret);
+			goto err;
+		}
+		io_uring_cqe_seen(&ring, cqe);
+		if (!memchr(ptr, c, cqe->res)) {
+			fprintf(stderr, "fail buffer check loop %d\n", i);
+			goto err;
+		}
+		c++;
+		ptr += cqe->res;
+	}
+
+	io_uring_free_buf_ring(&ring, br, 32, BUF_BGID);
+	io_uring_queue_exit(&ring);
+out:
+	free(buf);
+	unlink(fname);
+	return tret;
+err:
+	unlink(fname);
+	return T_EXIT_FAIL;
+}

data/vendor/liburing/test/read-write.c

@@ -46,7 +46,7 @@ static int create_nonaligned_buffers(void)
 
 static int _test_io(const char *file, struct io_uring *ring, int write,
 		    int buffered, int sqthread, int fixed, int nonvec,
-		    int buf_select, int seq, int exp_len)
+		    int buf_select, int seq, int exp_len, bool worker_offload)
 {
 	struct io_uring_sqe *sqe;
 	struct io_uring_cqe *cqe;
@@ -144,6 +144,10 @@ static int _test_io(const char *file, struct io_uring *ring, int write,
 		sqe->user_data = i;
 		if (sqthread)
 			sqe->flags |= IOSQE_FIXED_FILE;
+
+		if (worker_offload)
+			sqe->flags |= IOSQE_ASYNC;
+
 		if (buf_select) {
 			if (nonvec)
 				sqe->addr = 0;
@@ -227,12 +231,12 @@ err:
 
 static int __test_io(const char *file, struct io_uring *ring, int write,
 		     int buffered, int sqthread, int fixed, int nonvec,
-		     int buf_select, int seq, int exp_len)
+		     int buf_select, int seq, int exp_len, bool worker_offload)
 {
 	int ret;
 
 	ret = _test_io(file, ring, write, buffered, sqthread, fixed, nonvec,
-		       buf_select, seq, exp_len);
+		       buf_select, seq, exp_len, worker_offload);
 	if (ret)
 		return ret;
 
@@ -263,7 +267,7 @@ static int __test_io(const char *file, struct io_uring *ring, int write,
 			return ret;
 		}
 		ret = _test_io(file, &ring2, write, buffered, sqthread, 2,
-			       nonvec, buf_select, seq, exp_len);
+			       nonvec, buf_select, seq, exp_len, worker_offload);
 		if (ret)
 			return ret;
 
@@ -284,7 +288,7 @@ static int __test_io(const char *file, struct io_uring *ring, int write,
 }
 
 static int test_io(const char *file, int write, int buffered, int sqthread,
-		   int fixed, int nonvec, int exp_len)
+		   int fixed, int nonvec, int exp_len, bool worker_offload)
 {
 	struct io_uring ring;
 	int ret, ring_flags = 0;
@@ -301,7 +305,7 @@ static int test_io(const char *file, int write, int buffered, int sqthread,
 	}
 
 	ret = __test_io(file, &ring, write, buffered, sqthread, fixed, nonvec,
-			0, 0, exp_len);
+			0, 0, exp_len, worker_offload);
 	io_uring_queue_exit(&ring);
 	return ret;
 }
@@ -481,7 +485,8 @@ static int test_buf_select_short(const char *filename, int nonvec)
 		io_uring_cqe_seen(&ring, cqe);
 	}
 
-	ret = __test_io(filename, &ring, 0, 0, 0, 0, nonvec, 1, 1, exp_len);
+	ret = __test_io(filename, &ring, 0, 0, 0, 0, nonvec, 1, 1, exp_len,
+			false);
 
 	io_uring_queue_exit(&ring);
 	return ret;
@@ -620,7 +625,7 @@ static int test_buf_select(const char *filename, int nonvec)
 	for (i = 0; i < BUFFERS; i++)
 		memset(vecs[i].iov_base, i, vecs[i].iov_len);
 
-	ret = __test_io(filename, &ring, 1, 0, 0, 0, 0, 0, 1, BS);
+	ret = __test_io(filename, &ring, 1, 0, 0, 0, 0, 0, 1, BS, false);
 	if (ret) {
 		fprintf(stderr, "failed writing data\n");
 		return 1;
@@ -633,7 +638,7 @@ static int test_buf_select(const char *filename, int nonvec)
 	if (ret)
 		return ret;
 
-	ret = __test_io(filename, &ring, 0, 0, 0, 0, nonvec, 1, 1, BS);
+	ret = __test_io(filename, &ring, 0, 0, 0, 0, nonvec, 1, 1, BS, false);
 	io_uring_queue_exit(&ring);
 	return ret;
 }
@@ -933,17 +938,18 @@ int main(int argc, char *argv[])
 	vecs = t_create_buffers(BUFFERS, BS);
 
 	/* if we don't have nonvec read, skip testing that */
-	nr = has_nonvec_read() ? 32 : 16;
+	nr = has_nonvec_read() ? 64 : 32;
 
 	for (i = 0; i < nr; i++) {
 		int write = (i & 1) != 0;
 		int buffered = (i & 2) != 0;
 		int sqthread = (i & 4) != 0;
 		int fixed = (i & 8) != 0;
-		int nonvec = (i & 16) != 0;
+		int offload = (i & 16) != 0;
+		int nonvec = (i & 32) != 0;
 
 		ret = test_io(fname, write, buffered, sqthread, fixed, nonvec,
-			      BS);
+			      BS, offload);
 		if (ret) {
 			fprintf(stderr, "test_io failed %d/%d/%d/%d/%d\n",
 				write, buffered, sqthread, fixed, nonvec);
@@ -1047,14 +1053,15 @@ int main(int argc, char *argv[])
 		int buffered = (i & 2) != 0;
 		int sqthread = (i & 4) != 0;
 		int fixed = (i & 8) != 0;
-		int nonvec = (i & 16) != 0;
+		int offload = (i & 16) != 0;
+		int nonvec = (i & 32) != 0;
 
 		/* direct IO requires alignment, skip it */
 		if (!buffered || !fixed || nonvec)
 			continue;
 
 		ret = test_io(fname, write, buffered, sqthread, fixed, nonvec,
-			      -1);
+			      -1, offload);
 		if (ret) {
 			fprintf(stderr, "test_io failed %d/%d/%d/%d/%d\n",
 				write, buffered, sqthread, fixed, nonvec);