uringmachine 0.10 → 0.11.1

data/vendor/liburing/test/reg-wait.c

@@ -9,128 +9,67 @@
 #include <string.h>
 #include <fcntl.h>
 #include <sys/time.h>
+#include <sys/mman.h>
+#include <linux/mman.h>
 
 #include "liburing.h"
 #include "helpers.h"
 #include "test.h"
+#include "../src/syscall.h"
 
-static struct io_uring_reg_wait *reg;
-
-static int test_invalid_reg2(void)
-{
-	struct io_uring ring;
-	void *buf, *ptr;
-	int ret;
-
-	io_uring_queue_init(1, &ring, 0);
-
-	if (posix_memalign(&buf, 4096, 4096))
-		return T_EXIT_FAIL;
-	memset(buf, 0, 4096);
-	ptr = buf + 4096 - 32;
+static const struct io_uring_reg_wait brief_wait = {
+	.flags = IORING_REG_WAIT_TS,
+	.ts.tv_sec = 0,
+	.ts.tv_nsec = 1000,
+};
 
-	ret = io_uring_register_wait_reg(&ring, ptr, 1);
-	if (ret != -EINVAL) {
-		fprintf(stderr, "register cqwait: %d\n", ret);
-		return T_EXIT_FAIL;
-	}
-
-	ptr = buf + (sizeof(struct io_uring_reg_wait) / 2);
-	ret = io_uring_register_wait_reg(&ring, ptr, 1);
-	if (ret != -EINVAL) {
-		fprintf(stderr, "register cqwait: %d\n", ret);
-		return T_EXIT_FAIL;
-	}
-
-	free(buf);
-	buf = (void *) 0x1000;
-	ret = io_uring_register_wait_reg(&ring, buf, 1);
-	if (ret != -EFAULT) {
-		fprintf(stderr, "register cqwait: %d\n", ret);
-		return T_EXIT_FAIL;
-	}
-
-	buf = (void *) 0x1240;
-	ret = io_uring_register_wait_reg(&ring, buf, 1);
-	if (ret != -EFAULT) {
-		fprintf(stderr, "register cqwait: %d\n", ret);
-		return T_EXIT_FAIL;
-	}
-
-	buf = (void *) 0x1241;
-	ret = io_uring_register_wait_reg(&ring, buf, 1);
-	if (ret != -EINVAL) {
-		fprintf(stderr, "register cqwait: %d\n", ret);
-		return T_EXIT_FAIL;
-	}
+static bool has_kernel_regions;
 
-	io_uring_queue_exit(&ring);
-	return T_EXIT_PASS;
+static int test_wait_reg_offset(struct io_uring *ring,
+				 unsigned wait_nr, unsigned long offset)
+{
+	return __sys_io_uring_enter2(ring->ring_fd, 0, wait_nr,
+				     IORING_ENTER_GETEVENTS |
+				     IORING_ENTER_EXT_ARG |
+				     IORING_ENTER_EXT_ARG_REG,
+				     (void *)offset,
+				     sizeof(struct io_uring_reg_wait));
 }
 
-static int test_invalid_reg(void)
+static int __init_ring_with_region(struct io_uring *ring, unsigned ring_flags,
+				   struct io_uring_mem_region_reg *pr,
+				   bool disabled)
 {
-	struct io_uring_reg_wait *ireg;
-	struct io_uring_cqe *cqe;
-	struct io_uring ring;
-	struct timeval tv;
-	void *buf, *ptr;
+	int flags = disabled ? IORING_SETUP_R_DISABLED : 0;
 	int ret;
 
-	io_uring_queue_init(1, &ring, 0);
-
-	if (posix_memalign(&buf, 4096, 4096))
-		return T_EXIT_FAIL;
-	memset(buf, 0, 4096);
-	ptr = buf + 512;
-	ireg = ptr;
-
-	ret = io_uring_register_wait_reg(&ring, ireg, 56);
+	ret = io_uring_queue_init(8, ring, flags);
 	if (ret) {
-		fprintf(stderr, "register cqwait: %d\n", ret);
-		return T_EXIT_FAIL;
+		if (ret != -EINVAL)
+			fprintf(stderr, "ring setup failed: %d\n", ret);
+		return ret;
 	}
 
-	ireg = ptr;
-	memset(ireg, 0, sizeof(*ireg));
-	ireg->ts.tv_sec = 1;
-	ireg->ts.tv_nsec = 0;
-	ireg->flags = IORING_REG_WAIT_TS;
-
-	gettimeofday(&tv, NULL);
-	ret = io_uring_submit_and_wait_reg(&ring, &cqe, 1, 0);
-	if (ret != -ETIME) {
-		fprintf(stderr, "wait_reg failed: %d\n", ret);
-		return T_EXIT_FAIL;
-	}
-
-	ret = mtime_since_now(&tv);
-	/* allow some slack, should be around 1.1s */
-	if (ret < 1000 || ret > 1200) {
-		fprintf(stderr, "wait too long or short: %d\n", ret);
+	ret = io_uring_register_region(ring, pr);
+	if (ret)
 		goto err;
-	}
 
-	memset(ireg, 0, sizeof(*ireg));
-	ireg->ts.tv_sec = 1;
-	ireg->ts.tv_nsec = 0;
-	ireg->flags = IORING_REG_WAIT_TS;
-
-	gettimeofday(&tv, NULL);
-	ret = io_uring_submit_and_wait_reg(&ring, &cqe, 1, 56);
-	if (ret != -EFAULT) {
-		fprintf(stderr, "out-of-range reg_wait failed: %d\n", ret);
-		return T_EXIT_FAIL;
+	if (disabled) {
+		ret = io_uring_enable_rings(ring);
+		if (ret) {
+			fprintf(stderr, "io_uring_enable_rings failure %i\n", ret);
+			goto err;
+		}
 	}
-
-	free(buf);
-	io_uring_queue_exit(&ring);
-	return T_EXIT_PASS;
+	return 0;
 err:
-	io_uring_queue_exit(&ring);
-	return T_EXIT_FAIL;
+	io_uring_queue_exit(ring);
+	return ret;
 }
 
+static int page_size;
+static struct io_uring_reg_wait *reg;
+
 static int test_invalid_sig(struct io_uring *ring)
 {
 	struct io_uring_cqe *cqe;
@@ -164,6 +103,70 @@ static int test_invalid_sig(struct io_uring *ring)
 	return T_EXIT_PASS;
 }
 
+static int test_offsets(struct io_uring *ring, struct io_uring_reg_wait *base,
+			size_t size, bool overallocated)
+{
+	struct io_uring_cqe *cqe;
+	int max_index = size / sizeof(struct io_uring_reg_wait);
+	struct io_uring_reg_wait *rw;
+	unsigned long offset;
+	int copy_size;
+	void *rw_ptr;
+	int ret;
+
+	rw = base;
+	memcpy(rw, &brief_wait, sizeof(brief_wait));
+	ret = io_uring_submit_and_wait_reg(ring, &cqe, 1, 0);
+	if (ret != -ETIME) {
+		fprintf(stderr, "0 index failed: %d\n", ret);
+		return T_EXIT_FAIL;
+	}
+
+	if (overallocated) {
+		rw = base + max_index;
+		memcpy(rw, &brief_wait, sizeof(brief_wait));
+	}
+	ret = io_uring_submit_and_wait_reg(ring, &cqe, 1, max_index);
+	if (ret != -EFAULT) {
+		fprintf(stderr, "max+1 index failed: %d\n", ret);
+		return T_EXIT_FAIL;
+	}
+
+	rw = base + max_index - 1;
+	memcpy(rw, &brief_wait, sizeof(brief_wait));
+	ret = io_uring_submit_and_wait_reg(ring, &cqe, 1, max_index - 1);
+	if (ret != -ETIME) {
+		fprintf(stderr, "last index failed: %d\n", ret);
+		return T_EXIT_FAIL;
+	}
+
+	offset = 0UL - sizeof(long);
+	ret = test_wait_reg_offset(ring, 1, offset);
+	if (ret != -EFAULT) {
+		fprintf(stderr, "overflow offset failed: %d\n", ret);
+		return T_EXIT_FAIL;
+	}
+
+	offset = size - sizeof(long);
+	rw = (void *)base + offset;
+	copy_size = overallocated ? sizeof(brief_wait) : sizeof(long);
+	memcpy(rw, &brief_wait, copy_size);
+
+	ret = test_wait_reg_offset(ring, 1, offset);
+	if (ret != -EFAULT) {
+		fprintf(stderr, "OOB offset failed: %d\n", ret);
+		return T_EXIT_FAIL;
+	}
+
+	offset = 1;
+	rw_ptr = (void *) base + offset;
+	memcpy(rw_ptr, &brief_wait, sizeof(brief_wait));
+	/* undefined behaviour, check the kernel doesn't crash */
+	(void)test_wait_reg_offset(ring, 1, offset);
+
+	return 0;
+}
+
 static int test_basic(struct io_uring *ring)
 {
 	struct io_uring_cqe *cqe;
@@ -192,27 +195,50 @@ err:
 	return T_EXIT_FAIL;
 }
 
-static int test_ring(void)
+static int test_wait_arg(void)
 {
+	struct io_uring_region_desc rd = {};
+	struct io_uring_mem_region_reg mr = {};
 	struct io_uring ring;
-	struct io_uring_params p = { };
+	void *buffer;
 	int ret;
 
-	p.flags = 0;
-	ret = io_uring_queue_init_params(8, &ring, &p);
+	ret = io_uring_queue_init(8, &ring, IORING_SETUP_R_DISABLED);
 	if (ret) {
+		if (ret == -EINVAL) {
+			printf("IORING_SETUP_R_DISABLED not supported, skip\n");
+			return 0;
+		}
 		fprintf(stderr, "ring setup failed: %d\n", ret);
+		return T_EXIT_FAIL;
+	}
+
+	buffer = aligned_alloc(page_size, page_size * 4);
+	if (!buffer) {
+		fprintf(stderr, "allocation failed\n");
+		return T_EXIT_FAIL;
+	}
+
+	rd.user_addr = (__u64)(unsigned long)buffer;
+	rd.size = page_size;
+	rd.flags = IORING_MEM_REGION_TYPE_USER;
+	mr.region_uptr = (__u64)(unsigned long)&rd;
+	mr.flags = IORING_MEM_REGION_REG_WAIT_ARG;
+
+	ret = io_uring_register_region(&ring, &mr);
+	if (ret) {
+		fprintf(stderr, "region reg failed %i\n", ret);
 		return 1;
 	}
 
-	reg = io_uring_setup_reg_wait(&ring, 64, &ret);
-	if (!reg) {
-		if (ret == -EINVAL)
-			return T_EXIT_SKIP;
-		fprintf(stderr, "setup_reg_wait: %d\n", ret);
+	ret = io_uring_enable_rings(&ring);
+	if (ret) {
+		fprintf(stderr, "io_uring_enable_rings failure %i\n", ret);
 		return T_EXIT_FAIL;
 	}
 
+	reg = buffer;
+
 	ret = test_basic(&ring);
 	if (ret == T_EXIT_FAIL) {
 		fprintf(stderr, "test failed\n");
@@ -225,27 +251,330 @@ static int test_ring(void)
 		goto err;
 	}
 
-	ret = test_invalid_reg();
+	ret = test_offsets(&ring, buffer, page_size, true);
 	if (ret == T_EXIT_FAIL) {
-		fprintf(stderr, "test_invalid_reg failed\n");
+		fprintf(stderr, "test_offsets failed\n");
 		goto err;
 	}
-
-	ret = test_invalid_reg2();
-	if (ret == T_EXIT_FAIL) {
-		fprintf(stderr, "test_invalid_reg2 failed\n");
-		goto err;
-	}
-
 err:
+	free(buffer);
 	io_uring_queue_exit(&ring);
 	return ret;
 }
 
+static int test_try_register_region(struct io_uring_mem_region_reg *pr,
+				    bool disabled)
+{
+	struct io_uring ring;
+	int ret;
+
+	ret = __init_ring_with_region(&ring, 0, pr, disabled);
+	if (!ret)
+		io_uring_queue_exit(&ring);
+	return ret;
+}
+
+static int test_regions(void)
+{
+	struct io_uring_region_desc rd = {};
+	struct io_uring_mem_region_reg mr = {};
+	void *buffer;
+	int ret;
+
+	buffer = aligned_alloc(page_size, page_size * 4);
+	if (!buffer) {
+		fprintf(stderr, "allocation failed\n");
+		return T_EXIT_FAIL;
+	}
+
+	rd.user_addr = (__u64)(unsigned long)buffer;
+	rd.size = page_size;
+	rd.flags = IORING_MEM_REGION_TYPE_USER;
+
+	mr.region_uptr = (__u64)(unsigned long)&rd;
+	mr.flags = IORING_MEM_REGION_REG_WAIT_ARG;
+
+	ret = test_try_register_region(&mr, true);
+	if (ret == -EINVAL) {
+		free(buffer);
+		return T_EXIT_SKIP;
+	}
+	if (ret) {
+		fprintf(stderr, "region: register normal fail %i\n", ret);
+		return T_EXIT_FAIL;
+	}
+
+	ret = test_try_register_region(&mr, false);
+	if (ret != -EINVAL) {
+		fprintf(stderr, "region: register with !R_DISABLED fail %i\n", ret);
+		return T_EXIT_FAIL;
+	}
+
+	rd.size = page_size * 4;
+	ret = test_try_register_region(&mr, true);
+	if (ret) {
+		fprintf(stderr, "test_try_register_region() 16KB fail %i\n", ret);
+		return T_EXIT_FAIL;
+	}
+	rd.size = page_size;
+
+	rd.user_addr = 0;
+	ret = test_try_register_region(&mr, true);
+	if (ret != -EFAULT) {
+		fprintf(stderr, "test_try_register_region() null uptr fail %i\n", ret);
+		return T_EXIT_FAIL;
+	}
+	rd.user_addr = (__u64)(unsigned long)buffer;
+
+	rd.flags = 0;
+	ret = test_try_register_region(&mr, true);
+	if (!ret) {
+		fprintf(stderr, "test_try_register_region() kernel alloc with uptr fail %i\n", ret);
+		return T_EXIT_FAIL;
+	}
+	rd.flags = IORING_MEM_REGION_TYPE_USER;
+
+	rd.size = 0;
+	ret = test_try_register_region(&mr, true);
+	if (!ret) {
+		fprintf(stderr, "test_try_register_region() 0-size fail %i\n", ret);
+		return T_EXIT_FAIL;
+	}
+	rd.size = page_size;
+
+	mr.region_uptr = 0;
+	ret = test_try_register_region(&mr, true);
+	if (!ret) {
+		fprintf(stderr, "test_try_register_region() NULL region %i\n", ret);
+		return T_EXIT_FAIL;
+	}
+	mr.region_uptr = (__u64)(unsigned long)&rd;
+
+	rd.user_addr += 16;
+	ret = test_try_register_region(&mr, true);
+	if (!ret) {
+		fprintf(stderr, "test_try_register_region() misaligned region %i\n", ret);
+		return T_EXIT_FAIL;
+	}
+
+	rd.user_addr = 0x1000;
+	ret = test_try_register_region(&mr, true);
+	if (!ret) {
+		fprintf(stderr, "test_try_register_region() bogus uptr %i\n", ret);
+		return T_EXIT_FAIL;
+	}
+	rd.user_addr = (__u64)(unsigned long)buffer;
+	free(buffer);
+
+	buffer = mmap(NULL, page_size, PROT_READ, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
+	if (buffer == MAP_FAILED) {
+		fprintf(stderr, "mmap alloc failed\n");
+		return 1;
+	}
+
+	rd.user_addr = (__u64)(unsigned long)buffer;
+	ret = test_try_register_region(&mr, true);
+	if (ret != -EFAULT) {
+		fprintf(stderr, "test_try_register_region() RO uptr %i\n", ret);
+		return T_EXIT_FAIL;
+	}
+
+	rd.flags = 0;
+	rd.user_addr = 0;
+	ret = test_try_register_region(&mr, true);
+	if (ret == -EINVAL) {
+		has_kernel_regions = false;
+		goto out;
+	}
+	if (ret) {
+		fprintf(stderr, "test_try_register_region() failed kernel alloc %i\n", ret);
+		return T_EXIT_FAIL;
+	}
+
+	has_kernel_regions = true;
+	rd.flags = 0;
+	rd.user_addr = (__u64)(unsigned long)buffer;
+	ret = test_try_register_region(&mr, true);
+	if (!ret) {
+		fprintf(stderr, "test_try_register_region() failed uptr w kernel alloc %i\n", ret);
+		return T_EXIT_FAIL;
+	}
+out:
+	munmap(buffer, page_size);
+	return 0;
+}
+
+struct t_region {
+	void *ptr;
+	bool user_mem;
+	size_t size;
+};
+
+static void t_region_free(struct t_region *r)
+{
+	if (r->ptr)
+		munmap(r->ptr, r->size);
+}
+
+static int t_region_create_kernel(struct t_region *r,
+				  struct io_uring *ring)
+{
+	struct io_uring_region_desc rd = { .size = r->size, };
+	struct io_uring_mem_region_reg mr = {
+		.region_uptr = (__u64)(unsigned long)&rd,
+		.flags = IORING_MEM_REGION_REG_WAIT_ARG,
+	};
+	void *p;
+	int ret;
+
+	ret = io_uring_register_region(ring, &mr);
+	if (ret)
+		return ret;
+
+	p = mmap(NULL, r->size, PROT_READ | PROT_WRITE,
+		 MAP_SHARED | MAP_POPULATE, ring->ring_fd, rd.mmap_offset);
+	if (p == MAP_FAILED)
+		return -EFAULT;
+
+	r->ptr = p;
+	r->user_mem = false;
+	return 0;
+}
+
+static int t_region_create_user(struct t_region *r,
+				struct io_uring *ring,
+				bool huge)
+{
+	struct io_uring_region_desc rd = {};
+	struct io_uring_mem_region_reg mr = {};
+	int flags = MAP_PRIVATE | MAP_ANONYMOUS;
+	void *p;
+	int ret;
+
+	if (huge)
+		flags |= MAP_HUGETLB | MAP_HUGE_2MB;
+
+	p = mmap(NULL, r->size, PROT_READ | PROT_WRITE, flags, -1, 0);
+	if (p == MAP_FAILED)
+		return -ENOMEM;
+
+	mr.region_uptr = (__u64)(unsigned long)&rd;
+	mr.flags = IORING_MEM_REGION_REG_WAIT_ARG;
+	rd.user_addr = (__u64)(unsigned long)p;
+	rd.flags = IORING_MEM_REGION_TYPE_USER;
+	rd.size = r->size;
+
+	ret = io_uring_register_region(ring, &mr);
+	if (ret) {
+		munmap(p, r->size);
+		return ret;
+	}
+	r->ptr = p;
+	r->user_mem = true;
+	return 0;
+}
+
+struct test_param {
+	size_t size;
+	bool huge_page;
+	bool kern_buf;
+};
+
+static int test_region_buffer_types(void)
+{
+	const size_t huge_size = 1024 * 1024 * 2;
+	struct test_param params[] = {
+		{ .size = page_size },
+		/* forcing vmap */
+		{ .size = page_size * 2 },
+		{ .size = page_size * 16 },
+		/* huge page w/o vmap */
+		{ .size = huge_size, .huge_page = true },
+		/* huge page w/ vmap */
+		{ .size = huge_size * 2, .huge_page = true },
+		{ .size = page_size, .kern_buf = true },
+		/* likely to be a compound page */
+		{ .size = page_size * 2, .kern_buf = true },
+		{ .size = page_size * 8, .kern_buf = true },
+		/* kernel allocation + vmap */
+		{ .size = page_size * 512, .kern_buf = true },
+	};
+	struct io_uring ring;
+	int i, ret;
+
+	for (i = 0; i < ARRAY_SIZE(params); i++) {
+		struct t_region r = { .size = params[i].size, };
+
+		ret = io_uring_queue_init(8, &ring, IORING_SETUP_R_DISABLED);
+		if (ret) {
+			fprintf(stderr, "ring setup failed: %d\n", ret);
+			return ret;
+		}
+
+		if (params[i].kern_buf)
+			ret = t_region_create_kernel(&r, &ring);
+		else
+			ret = t_region_create_user(&r, &ring, params[i].huge_page);
+		if (ret) {
+			io_uring_queue_exit(&ring);
+			if (ret == -ENOMEM || ret == -EINVAL)
+				continue;
+			fprintf(stderr, "t_region_create_user failed, idx %i\n", i);
+			return 1;
+		}
+
+		ret = io_uring_enable_rings(&ring);
+		if (ret) {
+			fprintf(stderr, "io_uring_enable_rings failure %i\n", ret);
+			return ret;
+		}
+
+		ret = test_offsets(&ring, r.ptr, r.size, false);
+		if (ret) {
+			fprintf(stderr, "test_offsets failed, idx %i\n", i);
+			return 1;
+		}
+
+		t_region_free(&r);
+		io_uring_queue_exit(&ring);
+	}
+
+	return 0;
+}
+
 int main(int argc, char *argv[])
 {
+	int ret;
+
 	if (argc > 1)
 		return 0;
 
-	return test_ring();
+	page_size = sysconf(_SC_PAGESIZE);
+	if (page_size < 0) {
+		perror("sysconf(_SC_PAGESIZE)");
+		return 1;
+	}
+
+	ret = test_regions();
+	if (ret == T_EXIT_SKIP) {
+		printf("regions are not supported, skip\n");
+		return 0;
+	} else if (ret) {
+		fprintf(stderr, "test_region failed\n");
+		return 1;
+	}
+
+	ret = test_wait_arg();
+	if (ret == T_EXIT_FAIL) {
+		fprintf(stderr, "test_wait_arg failed\n");
+		return 1;
+	}
+
+	ret = test_region_buffer_types();
+	if (ret == T_EXIT_FAIL) {
+		fprintf(stderr, "test_region_buffer_types failed\n");
+		return 1;
+	}
+
+	return 0;
 }

data/vendor/liburing/test/regbuf-clone.c

@@ -305,6 +305,9 @@ static int test_offsets(void)
 		return T_EXIT_FAIL;
 	}
 
+	for (i = 0; i < NR_VECS; i++)
+		free(vecs[i].iov_base);
+
 	return T_EXIT_PASS;
 }
 
@@ -576,6 +579,48 @@ skip:
 	return res;
 }
 
+static int test_same(void)
+{
+	struct iovec vecs[2] = { };
+	struct io_uring src;
+	int ret;
+
+	ret = io_uring_queue_init(1, &src, 0);
+	if (ret) {
+		fprintf(stderr, "ring_init: %d\n", ret);
+		return T_EXIT_FAIL;
+	}
+
+	if (posix_memalign(&vecs[0].iov_base, 4096, BUF_SIZE))
+		return T_EXIT_SKIP;
+	vecs[0].iov_len = BUF_SIZE;
+
+	vecs[1].iov_base = NULL;
+	vecs[1].iov_len = 0;
+
+	ret = io_uring_register_buffers(&src, vecs, 2);
+	if (ret) {
+		fprintf(stderr, "reg buffers: %d\n", ret);
+		return T_EXIT_FAIL;
+	}
+
+	ret = io_uring_clone_buffers_offset(&src, &src, 1, 0, 2, IORING_REGISTER_DST_REPLACE);
+	if (ret) {
+		fprintf(stderr, "clone offset: %d\n", ret);
+		return T_EXIT_FAIL;
+	}
+
+	ret = io_uring_unregister_buffers(&src);
+	if (ret) {
+		fprintf(stderr, "rsc unregister buffers: %d\n", ret);
+		return T_EXIT_FAIL;
+	}
+
+	free(vecs[0].iov_base);
+	io_uring_queue_exit(&src);
+	return T_EXIT_PASS;
+}
+
 int main(int argc, char *argv[])
 {
 	int ret;
@@ -641,5 +686,13 @@ int main(int argc, char *argv[])
 	if (no_buf_offset)
 		return T_EXIT_PASS;
 
+	ret = test_same();
+	if (ret == T_EXIT_SKIP) {
+		return T_EXIT_PASS;
+	} else if (ret != T_EXIT_PASS) {
+		fprintf(stderr, "test_same failed\n");
+		return T_EXIT_FAIL;
+	}
+
 	return T_EXIT_PASS;
 }