uringmachine 0.16 → 0.18

data/vendor/liburing/test/init-mem.c

@@ -28,32 +28,40 @@ struct ctx {
 	unsigned long long	*post;
 };
 
+static int page_size;
+
 struct q_entries {
 	unsigned int sqes;
 	unsigned int cqes;
+	unsigned int ring_flags;
 };
 
 static int setup_ctx(struct ctx *ctx, struct q_entries *q)
 {
-	struct io_uring_params p = { };
-	int ret;
+	struct io_uring_params p = { .flags = q->ring_flags, };
+	int ret, rret;
 
-	if (posix_memalign(&ctx->mem, 4096, 2*1024*1024))
+	if (posix_memalign(&ctx->mem, page_size, 2*1024*1024))
 		return T_EXIT_FAIL;
 
 	memset(ctx->mem, 0, 2*1024*1024);
 
-	ctx->pre = ctx->mem + 4096 - sizeof(unsigned long long);
+	ctx->pre = ctx->mem + page_size - sizeof(unsigned long long);
 	*ctx->pre = PRE_RED;
 
-	ctx->ring_mem = ctx->mem + 4096;
+	ctx->ring_mem = ctx->mem + page_size;
 	p.flags |= IORING_SETUP_CQSIZE | IORING_SETUP_NO_SQARRAY;
 	p.sq_entries = q->sqes;
 	p.cq_entries = q->cqes;
 
+	rret = io_uring_memory_size_params(q->sqes, &p);
+	if (rret > 2 * 1024 * 1024) {
+		fprintf(stderr, "Alloc size too small\n");
+		return T_EXIT_FAIL;
+	}
+
 	ret = io_uring_queue_init_mem(q->sqes, &ctx->ring, &p,
 					ctx->ring_mem, 2*1024*1024);
-
 	if (ret < 0) {
 		if (ret == -EINVAL)
 			return T_EXIT_SKIP;
@@ -61,6 +69,12 @@ static int setup_ctx(struct ctx *ctx, struct q_entries *q)
 		return T_EXIT_FAIL;
 	}
 
+	if (ret != rret) {
+		fprintf(stderr, "Used mem %d differs from required %ld\n",
+				ret, (long) rret);
+		return T_EXIT_FAIL;
+	}
+
 	ctx->post = ctx->ring_mem + ret;
 	*ctx->post = POST_RED;
 	return 0;
@@ -141,18 +155,42 @@ err:
 int main(int argc, char *argv[])
 {
 	struct q_entries q_entries[] = {
-		{ 256, 16384 },
-		{ 32, 4096 },
-		{ 128, 8192 },
-		{ 4096, 32768 },
-		{ 1, 8 },
-		{ 2, 1024 },
+		{ 256, 16384, 0 },
+		{ 32, 4096, 0 },
+		{ 128, 8192, 0 },
+		{ 4096, 32768, 0 },
+		{ 1, 8, 0 },
+		{ 2, 1024, 0 },
+		{ 256, 16384, IORING_SETUP_SQE128 },
+		{ 32, 4096, IORING_SETUP_SQE128},
+		{ 128, 8192, IORING_SETUP_SQE128},
+		{ 4096, 32768, IORING_SETUP_SQE128},
+		{ 1, 8, IORING_SETUP_SQE128},
+		{ 2, 1024, IORING_SETUP_SQE128},
+		{ 256, 16384, IORING_SETUP_CQE32 },
+		{ 32, 4096, IORING_SETUP_CQE32},
+		{ 128, 8192, IORING_SETUP_CQE32},
+		{ 4096, 32768, IORING_SETUP_CQE32},
+		{ 1, 8, IORING_SETUP_CQE32},
+		{ 2, 1024, IORING_SETUP_CQE32},
+		{ 256, 16384, IORING_SETUP_SQE128 | IORING_SETUP_CQE32 },
+		{ 32, 4096, IORING_SETUP_SQE128 | IORING_SETUP_CQE32},
+		{ 128, 8192, IORING_SETUP_SQE128 | IORING_SETUP_CQE32},
+		{ 4096, 32768, IORING_SETUP_SQE128 | IORING_SETUP_CQE32},
+		{ 1, 8, IORING_SETUP_SQE128 | IORING_SETUP_CQE32},
+		{ 2, 1024, IORING_SETUP_SQE128 | IORING_SETUP_CQE32},
 	};
 	int i, ret;
 
 	if (argc > 1)
 		return T_EXIT_SKIP;
 
+	page_size = sysconf(_SC_PAGESIZE);
+	if (page_size < 0) {
+		perror("sysconf(_SC_PAGESIZE)");
+		return 1;
+	}
+
 	for (i = 0; i < ARRAY_SIZE(q_entries); i++) {
 		ret = test(&q_entries[i]);
 		if (ret == T_EXIT_SKIP) {

data/vendor/liburing/test/io_uring_passthrough.c

@@ -70,7 +70,7 @@ static int fill_pattern(int tc)
 }
 
 static int __test_io(const char *file, struct io_uring *ring, int tc, int read,
-		     int sqthread, int fixed, int nonvec)
+		     int sqthread, int fixed, int nonvec, int async, int linked)
 {
 	struct io_uring_sqe *sqe;
 	struct io_uring_cqe *cqe;
@@ -121,6 +121,16 @@ static int __test_io(const char *file, struct io_uring *ring, int tc, int read,
 		unsigned int iovcnt = 1;
 		size_t total_len;
 
+		if (linked) {
+			sqe = io_uring_get_sqe(ring);
+			io_uring_prep_nop(sqe);
+			if (async)
+				sqe->flags |= IOSQE_ASYNC;
+			sqe->flags |= IOSQE_IO_LINK;
+			sqe->user_data = 0x1000;
+			submit_count++;
+		}
+
 		sqe = io_uring_get_sqe(ring);
 		if (!sqe) {
 			fprintf(stderr, "sqe get failed\n");
@@ -135,6 +145,8 @@ static int __test_io(const char *file, struct io_uring *ring, int tc, int read,
 			do_fixed = 0;
 		if (do_fixed)
 			sqe->buf_index = 0;
+		if (async)
+			sqe->flags |= IOSQE_ASYNC;
 		if (nonvec)
 			sqe->cmd_op = NVME_URING_CMD_IO;
 		else
@@ -194,6 +206,8 @@ static int __test_io(const char *file, struct io_uring *ring, int tc, int read,
 	}
 
 	for (i = 0; i < submit_count; i++) {
+		int is_link;
+
 		ret = io_uring_wait_cqe(ring, &cqe);
 		if (ret) {
 			fprintf(stderr, "wait_cqe=%d\n", ret);
@@ -211,7 +225,10 @@ static int __test_io(const char *file, struct io_uring *ring, int tc, int read,
 			fprintf(stderr, "cqe res %d, wanted 0\n", cqe->res);
 			goto err;
 		}
+		is_link = cqe->user_data == 0x1000;
 		io_uring_cqe_seen(ring, cqe);
+		if (is_link)
+			continue;
 		if (read) {
 			int index = cqe->user_data & 0xffffffff;
 			void *buf = vecs[index].iov_base;
@@ -248,7 +265,7 @@ err:
 }
 
 static int test_io(const char *file, int tc, int read, int sqthread,
-		   int fixed, int nonvec, int hybrid)
+		   int fixed, int nonvec, int hybrid, int async, int linked)
 {
 	struct io_uring ring;
 	int ret, ring_flags = 0;
@@ -265,7 +282,7 @@ static int test_io(const char *file, int tc, int read, int sqthread,
 	if (fixed && (!vec_fixed_supported && !nonvec))
 		return 0;
 
-	ret = t_create_ring(64, &ring, ring_flags);
+	ret = t_create_ring(128, &ring, ring_flags);
 	if (ret == T_SETUP_SKIP)
 		return 0;
 	if (ret != T_SETUP_OK) {
@@ -277,7 +294,7 @@ static int test_io(const char *file, int tc, int read, int sqthread,
 		return 1;
 	}
 
-	ret = __test_io(file, &ring, tc, read, sqthread, fixed, nonvec);
+	ret = __test_io(file, &ring, tc, read, sqthread, fixed, nonvec, async, linked);
 	io_uring_queue_exit(&ring);
 
 	return ret;
@@ -464,14 +481,16 @@ int main(int argc, char *argv[])
 		t_posix_memalign(&meta_mem, 0x1000,
 				 meta_size * BUFFERS * (BS >> lba_shift));
 
-	for (i = 0; i < 32; i++) {
+	for (i = 0; i < 64; i++) {
 		int read = (i & 1) != 0;
 		int sqthread = (i & 2) != 0;
 		int fixed = (i & 4) != 0;
 		int nonvec = (i & 8) != 0;
 		int hybrid = (i & 16) != 0;
+		int async = (i & 32) != 0;
+		int linked = (i & 64) != 0;
 
-		ret = test_io(fname, i, read, sqthread, fixed, nonvec, hybrid);
+		ret = test_io(fname, i, read, sqthread, fixed, nonvec, hybrid, async, linked);
 		if (no_pt)
 			break;
 		if (ret) {

data/vendor/liburing/test/msg-ring.c

@@ -15,13 +15,59 @@
 #include "helpers.h"
 
 static int no_msg;
+static int no_sync_msg;
 
-static int test_own(struct io_uring *ring)
+static int test_own_sync(struct io_uring *ring)
 {
+	struct io_uring_sqe sqe = { };
 	struct io_uring_cqe *cqe;
+	int ret;
+
+	if (no_sync_msg)
+		return 0;
+
+	io_uring_prep_msg_ring(&sqe, ring->ring_fd, 0x10, 0x1234, 0);
+	sqe.user_data = 1;
+
+	ret = io_uring_register_sync_msg(&sqe);
+	if (ret == -EINVAL) {
+		no_sync_msg = 1;
+		return 0;
+	} else if (ret != 0) {
+		fprintf(stderr, "register_sync_msg: %d\n", ret);
+		return 1;
+	}
+
+	ret = io_uring_wait_cqe(ring, &cqe);
+	if (ret < 0) {
+		fprintf(stderr, "wait completion %d\n", ret);
+		return 1;
+	}
+	switch (cqe->user_data) {
+	case 0x1234:
+		if (cqe->res != 0x10) {
+			fprintf(stderr, "invalid len %x\n", cqe->res);
+			return -1;
+		}
+		break;
+	default:
+		fprintf(stderr, "Invalid user_data\n");
+		return -1;
+	}
+
+	io_uring_cqe_seen(ring, cqe);
+	return 0;
+}
+
+static int test_own(struct io_uring *ring, int do_sync)
+{
 	struct io_uring_sqe *sqe;
+	struct io_uring_cqe *cqe;
 	int ret, i;
 
+	if (do_sync)
+		return test_own_sync(ring);
+
 	sqe = io_uring_get_sqe(ring);
 	if (!sqe) {
 		fprintf(stderr, "get sqe failed\n");
@@ -116,7 +162,47 @@ err:
 	return (void *) (unsigned long) 1;
 }
 
-static int test_remote(struct io_uring *ring, unsigned int ring_flags)
+static int test_remote_sync(unsigned int ring_flags)
+{
+	struct io_uring *target;
+	pthread_t thread;
+	void *tret;
+	struct io_uring_sqe sqe = { };
+	struct data d;
+	int ret;
+
+	if (no_sync_msg)
+		return 0;
+
+	d.flags = ring_flags;
+	pthread_barrier_init(&d.barrier, NULL, 2);
+	pthread_barrier_init(&d.startup, NULL, 2);
+	pthread_create(&thread, NULL, wait_cqe_fn, &d);
+
+	pthread_barrier_wait(&d.startup);
+	target = d.ring;
+
+	io_uring_prep_msg_ring(&sqe, target->ring_fd, 0x20, 0x5aa5, 0);
+	sqe.user_data = 1;
+
+	pthread_barrier_wait(&d.barrier);
+
+	ret = io_uring_register_sync_msg(&sqe);
+	if (ret == -EINVAL) {
+		no_sync_msg = 1;
+		return 0;
+	} else if (ret != 0) {
+		fprintf(stderr, "sync_msg: %d\n", ret);
+		goto err;
+	}
+	pthread_join(thread, &tret);
+	return 0;
+err:
+	return 1;
+}
+
+static int test_remote(struct io_uring *ring, unsigned int ring_flags,
+		       int do_sync)
 {
 	struct io_uring *target;
 	pthread_t thread;
@@ -126,6 +212,9 @@ static int test_remote(struct io_uring *ring, unsigned int ring_flags)
 	struct data d;
 	int ret;
 
+	if (do_sync)
+		return test_remote_sync(ring_flags);
+
 	d.flags = ring_flags;
 	pthread_barrier_init(&d.barrier, NULL, 2);
 	pthread_barrier_init(&d.startup, NULL, 2);
@@ -367,16 +456,29 @@ static int test(int ring_flags)
 		return T_EXIT_FAIL;
 	}
 
-	ret = test_own(&ring);
+	ret = test_own(&ring, 0);
 	if (ret) {
-		fprintf(stderr, "test_own failed\n");
+		fprintf(stderr, "test_own sync failed\n");
 		return T_EXIT_FAIL;
 	}
 	if (no_msg)
 		return T_EXIT_SKIP;
-	ret = test_own(&pring);
+
+	ret = test_own(&ring, 1);
+	if (ret) {
+		fprintf(stderr, "test_own async failed\n");
+		return T_EXIT_FAIL;
+	}
+
+	ret = test_own(&pring, 0);
+	if (ret) {
+		fprintf(stderr, "test_own async iopoll failed\n");
+		return T_EXIT_FAIL;
+	}
+
+	ret = test_own(&pring, 1);
 	if (ret) {
-		fprintf(stderr, "test_own iopoll failed\n");
+		fprintf(stderr, "test_own sync iopoll failed\n");
 		return T_EXIT_FAIL;
 	}
 
@@ -394,12 +496,18 @@ static int test(int ring_flags)
 		}
 	}
 
-	ret = test_remote(&ring, ring_flags);
+	ret = test_remote(&ring, ring_flags, 0);
 	if (ret) {
 		fprintf(stderr, "test_remote failed\n");
 		return T_EXIT_FAIL;
 	}
 
+	ret = test_remote(&ring, ring_flags, 1);
+	if (ret) {
+		fprintf(stderr, "test_remote sync failed\n");
+		return T_EXIT_FAIL;
+	}
+
 	io_uring_queue_exit(&ring);
 	io_uring_queue_exit(&pring);
 
@@ -411,9 +519,15 @@ static int test(int ring_flags)
 			return T_EXIT_FAIL;
 		}
 
-		ret = test_own(&ring);
+		ret = test_own(&ring, 0);
+		if (ret) {
+			fprintf(stderr, "test_own async deferred failed\n");
+			return T_EXIT_FAIL;
+		}
+
+		ret = test_own(&ring, 1);
 		if (ret) {
-			fprintf(stderr, "test_own deferred failed\n");
+			fprintf(stderr, "test_own sync deferred failed\n");
 			return T_EXIT_FAIL;
 		}
 

data/vendor/liburing/test/recv-bundle-short-ooo.c

@@ -142,6 +142,19 @@ static int setup_buf_ring(struct buf_ring_data *data, struct io_uring *ring,
 	return T_EXIT_PASS;
 }
 
+static void dump_buf(const char *msg, uint8_t *buf, int len)
+{
+	int i;
+
+	fprintf(stderr, "Buffer %s\n", msg);
+	for (i = 0; i < len; i++) {
+		fprintf(stderr, "%3x ", buf[i]);
+		if (i && !(i & 15))
+			fprintf(stderr, "\n");
+	}
+	fprintf(stderr, "\n");
+}
+
 /**
  * Verifies that received buffer data matches expected data
  *
@@ -157,7 +170,9 @@ static int verify_received_buffer(struct buf_data *buf, uint8_t *expected_data_s
 
 	for (uint32_t i = 0; i < buf->len; i++) {
 		if (data[i] != expected_data_start[i]) {
-			fprintf(stderr, "Recv data ordering mismatch\n");
+			fprintf(stderr, "Recv data ordering mismatch, offset %d\n", i);
+			dump_buf("expected", expected_data_start, buf->len);
+			dump_buf("received", data, buf->len);
 			return 1;
 		}
 	}