uringmachine 0.3 → 0.5

data/vendor/liburing/test/read-mshot.c

@@ -23,7 +23,134 @@
 
 #define NR_OVERFLOW	(NR_BUFS / 4)
 
-static int no_buf_ring, no_read_mshot;
+static int no_buf_ring, no_read_mshot, no_buf_ring_inc;
+
+static void arm_read(struct io_uring *ring, int fd, int use_mshot)
+{
+	struct io_uring_sqe *sqe;
+
+	sqe = io_uring_get_sqe(ring);
+	if (use_mshot) {
+		io_uring_prep_read_multishot(sqe, fd, 0, 0, BUF_BGID);
+	} else {
+		io_uring_prep_read(sqe, fd, NULL, 0, 0);
+		sqe->flags = IOSQE_BUFFER_SELECT;
+		sqe->buf_group = BUF_BGID;
+	}
+
+	io_uring_submit(ring);
+}
+
+static int test_inc(int use_mshot, int flags)
+{
+	struct io_uring_buf_ring *br;
+	struct io_uring_params p = { };
+	struct io_uring_cqe *cqe;
+	struct io_uring ring;
+	int nbytes = 65536;
+	int ret, fds[2], i;
+	char tmp[31];
+	char *buf;
+	void *ptr;
+	int bid = -1;
+	int bid_bytes;
+
+	if (no_buf_ring)
+		return 0;
+
+	p.flags = flags;
+	ret = io_uring_queue_init_params(64, &ring, &p);
+	if (ret) {
+		fprintf(stderr, "ring setup failed: %d\n", ret);
+		return 1;
+	}
+
+	if (pipe(fds) < 0) {
+		perror("pipe");
+		return 1;
+	}
+
+	if (posix_memalign((void **) &buf, 4096, 65536))
+		return 1;
+
+	br = io_uring_setup_buf_ring(&ring, 32, BUF_BGID, IOU_PBUF_RING_INC, &ret);
+	if (!br) {
+		if (ret == -EINVAL) {
+			no_buf_ring_inc = 1;
+			free(buf);
+			return 0;
+		}
+		fprintf(stderr, "Buffer ring register failed %d\n", ret);
+		return 1;
+	}
+
+	ptr = buf;
+	buf = ptr + 65536 - 2048;
+	for (i = 0; i < 32; i++) {
+		io_uring_buf_ring_add(br, buf, 2048, i, 31, i);
+		buf -= 2048;
+	}
+	io_uring_buf_ring_advance(br, 32);
+
+	memset(tmp, 0x5a, sizeof(tmp));
+
+	arm_read(&ring, fds[0], use_mshot);
+
+	bid_bytes = 0;
+	do {
+		int write_size = sizeof(tmp);
+
+		if (write_size > nbytes)
+			write_size = nbytes;
+
+		io_uring_get_events(&ring);
+		ret = io_uring_peek_cqe(&ring, &cqe);
+		if (!ret) {
+			int this_bid = cqe->flags >> IORING_CQE_BUFFER_SHIFT;
+			if (bid == -1) {
+				bid = this_bid;
+			} else if (bid != this_bid) {
+				if (bid_bytes != 2048) {
+					fprintf(stderr, "unexpected bid bytes %d\n",
+						bid_bytes);
+					return 1;
+				}
+				bid = this_bid;
+				bid_bytes = 0;
+			}
+			bid_bytes += cqe->res;
+			nbytes -= cqe->res;
+			if (!(cqe->flags & IORING_CQE_F_MORE))
+				arm_read(&ring, fds[0], use_mshot);
+			io_uring_cqe_seen(&ring, cqe);
+			if (!nbytes)
+				break;
+		}
+		usleep(1000);
+		ret = write(fds[1], tmp, write_size);
+		if (ret < 0) {
+			perror("write");
+			return 1;
+		} else if (ret != write_size) {
+			printf("short write %d\n", ret);
+			return 1;
+		}
+	} while (nbytes);
+
+	if (bid_bytes) {
+		if (bid_bytes != 2048) {
+			fprintf(stderr, "unexpected bid bytes %d\n", bid_bytes);
+			return 1;
+		}
+	}
+
+	io_uring_free_buf_ring(&ring, br, 32, BUF_BGID);
+	io_uring_queue_exit(&ring);
+	free(ptr);
+	close(fds[0]);
+	close(fds[1]);
+	return 0;
+}
 
 static int test_clamp(void)
 {
@@ -129,21 +256,24 @@ static int test_clamp(void)
 		io_uring_cqe_seen(&ring, cqe);
 	}
 
+	io_uring_free_buf_ring(&ring, br, NR_BUFS, BUF_BGID);
 	io_uring_queue_exit(&ring);
 	free(ptr);
 	return 0;
 }
 
-static int test(int first_good, int async, int overflow)
+static int test(int first_good, int async, int overflow, int incremental)
 {
 	struct io_uring_buf_ring *br;
 	struct io_uring_params p = { };
 	struct io_uring_sqe *sqe;
 	struct io_uring_cqe *cqe;
 	struct io_uring ring;
-	int ret, fds[2], i;
+	int ret, fds[2], i, start_msg = 0;
+	int br_flags = 0;
 	char tmp[32];
 	void *ptr[NR_BUFS];
+	char *inc_index;
 
 	p.flags = IORING_SETUP_CQSIZE;
 	if (!overflow)
@@ -156,14 +286,19 @@ static int test(int first_good, int async, int overflow)
 		return 1;
 	}
 
-	if (pipe(fds) < 0) {
-		perror("pipe");
-		return 1;
+	if (incremental) {
+		if (no_buf_ring_inc)
+			return 0;
+		br_flags |= IOU_PBUF_RING_INC;
 	}
 
-	br = io_uring_setup_buf_ring(&ring, NR_BUFS, BUF_BGID, 0, &ret);
+	br = io_uring_setup_buf_ring(&ring, NR_BUFS, BUF_BGID, br_flags, &ret);
 	if (!br) {
 		if (ret == -EINVAL) {
+			if (incremental) {
+				no_buf_ring_inc = 1;
+				return 0;
+			}
 			no_buf_ring = 1;
 			return 0;
 		}
@@ -171,17 +306,30 @@ static int test(int first_good, int async, int overflow)
 		return 1;
 	}
 
-	for (i = 0; i < NR_BUFS; i++) {
-		unsigned size = i <= 1 ? BUF_SIZE_FIRST : BUF_SIZE;
-		ptr[i] = malloc(size);
-		if (!ptr[i])
-			return 1;
-		io_uring_buf_ring_add(br, ptr[i], size, i + 1, BR_MASK, i);
+	if (pipe(fds) < 0) {
+		perror("pipe");
+		return 1;
+	}
+
+	if (!incremental) {
+		for (i = 0; i < NR_BUFS; i++) {
+			unsigned size = i <= 1 ? BUF_SIZE_FIRST : BUF_SIZE;
+			ptr[i] = malloc(size);
+			if (!ptr[i])
+				return 1;
+			io_uring_buf_ring_add(br, ptr[i], size, i + 1, BR_MASK, i);
+		}
+		inc_index = NULL;
+		io_uring_buf_ring_advance(br, NR_BUFS);
+	} else {
+		inc_index = ptr[0] = malloc(NR_BUFS * BUF_SIZE);
+		memset(inc_index, 0, NR_BUFS * BUF_SIZE);
+		io_uring_buf_ring_add(br, ptr[0], NR_BUFS * BUF_SIZE, 1, BR_MASK, 0);
+		io_uring_buf_ring_advance(br, 1);
 	}
-	io_uring_buf_ring_advance(br, NR_BUFS);
 
 	if (first_good) {
-		sprintf(tmp, "this is buffer %d\n", 0);
+		sprintf(tmp, "this is buffer %d\n", start_msg++);
 		ret = write(fds[1], tmp, strlen(tmp));
 	}
 
@@ -201,7 +349,7 @@ static int test(int first_good, int async, int overflow)
 	for (i = 0; i < NR_BUFS + !first_good; i++) {
 		/* prevent pipe buffer merging */
 		usleep(1000);
-		sprintf(tmp, "this is buffer %d\n", i + 1);
+		sprintf(tmp, "this is buffer %d\n", i + start_msg);
 		ret = write(fds[1], tmp, strlen(tmp));
 		if (ret != strlen(tmp)) {
 			fprintf(stderr, "write ret %d\n", ret);
@@ -210,6 +358,8 @@ static int test(int first_good, int async, int overflow)
 	}
 
 	for (i = 0; i < NR_BUFS + 1; i++) {
+		int bid;
+
 		ret = io_uring_wait_cqe(&ring, &cqe);
 		if (ret) {
 			fprintf(stderr, "wait cqe failed %d\n", ret);
@@ -234,6 +384,18 @@ static int test(int first_good, int async, int overflow)
 			fprintf(stderr, "no buffer selected\n");
 			return 1;
 		}
+		bid = cqe->flags >> IORING_CQE_BUFFER_SHIFT;
+		if (incremental && bid != 1) {
+			fprintf(stderr, "bid %d for incremental\n", bid);
+			return 1;
+		}
+		if (incremental && !first_good) {
+			char out_buf[64];
+			sprintf(out_buf, "this is buffer %d\n", i + start_msg);
+			if (strncmp(inc_index, out_buf, strlen(out_buf)))
+				return 1;
+			inc_index += cqe->res;
+		}
 		if (!(cqe->flags & IORING_CQE_F_MORE)) {
 			/* we expect this on overflow */
 			if (overflow && i >= NR_OVERFLOW)
@@ -249,9 +411,15 @@ static int test(int first_good, int async, int overflow)
 		io_uring_cqe_seen(&ring, cqe);
 	}
 
+
+	io_uring_free_buf_ring(&ring, br, NR_BUFS, BUF_BGID);
 	io_uring_queue_exit(&ring);
-	for (i = 0; i < NR_BUFS; i++)
-		free(ptr[i]);
+	if (incremental) {
+		free(ptr[0]);
+	} else {
+		for (i = 0; i < NR_BUFS; i++)
+			free(ptr[i]);
+	}
 	return 0;
 }
 
@@ -320,6 +488,7 @@ static int test_invalid(int async)
 	}
 
 	io_uring_cqe_seen(&ring, cqe);
+	io_uring_free_buf_ring(&ring, br, 1, BUF_BGID);
 	io_uring_queue_exit(&ring);
 	free(buf);
 	return 0;
@@ -332,56 +501,106 @@ int main(int argc, char *argv[])
 	if (argc > 1)
 		return T_EXIT_SKIP;
 
-	ret = test(0, 0, 0);
+	ret = test(0, 0, 0, 0);
 	if (ret) {
 		fprintf(stderr, "test 0 0 0 failed\n");
 		return T_EXIT_FAIL;
 	}
-	if (no_buf_ring || no_read_mshot)
+	if (no_buf_ring || no_read_mshot) {
+		printf("skip\n");
 		return T_EXIT_SKIP;
+	}
 
-	ret = test(0, 1, 0);
+	ret = test(0, 1, 0, 0);
 	if (ret) {
 		fprintf(stderr, "test 0 1 0, failed\n");
 		return T_EXIT_FAIL;
 	}
 
-	ret = test(1, 0, 0);
+	ret = test(1, 0, 0, 0);
 	if (ret) {
 		fprintf(stderr, "test 1 0 0 failed\n");
 		return T_EXIT_FAIL;
 	}
 
-	ret = test(0, 0, 1);
+	ret = test(0, 0, 1, 0);
 	if (ret) {
 		fprintf(stderr, "test 0 0 1 failed\n");
 		return T_EXIT_FAIL;
 	}
 
-	ret = test(0, 1, 1);
+	ret = test(0, 1, 1, 0);
 	if (ret) {
 		fprintf(stderr, "test 0 1 1 failed\n");
 		return T_EXIT_FAIL;
 	}
 
-	ret = test(1, 0, 1);
+	ret = test(1, 0, 1, 0);
 	if (ret) {
 		fprintf(stderr, "test 1 0 1, failed\n");
 		return T_EXIT_FAIL;
 	}
 
-	ret = test(1, 0, 1);
+	ret = test(1, 0, 1, 0);
 	if (ret) {
 		fprintf(stderr, "test 1 0 1 failed\n");
 		return T_EXIT_FAIL;
 	}
 
-	ret = test(1, 1, 1);
+	ret = test(1, 1, 1, 0);
 	if (ret) {
 		fprintf(stderr, "test 1 1 1 failed\n");
 		return T_EXIT_FAIL;
 	}
 
+	ret = test(0, 0, 0, 1);
+	if (ret) {
+		fprintf(stderr, "test 0 0 0 1 failed\n");
+		return T_EXIT_FAIL;
+	}
+
+	ret = test(0, 0, 1, 1);
+	if (ret) {
+		fprintf(stderr, "test 0 0 1 1 failed\n");
+		return T_EXIT_FAIL;
+	}
+
+	ret = test(0, 1, 0, 1);
+	if (ret) {
+		fprintf(stderr, "test 0 1 0 1 failed\n");
+		return T_EXIT_FAIL;
+	}
+
+	ret = test(0, 1, 1, 1);
+	if (ret) {
+		fprintf(stderr, "test 0 1 1 1 failed\n");
+		return T_EXIT_FAIL;
+	}
+
+	ret = test(1, 0, 0, 1);
+	if (ret) {
+		fprintf(stderr, "test 1 0 0 1 failed\n");
+		return T_EXIT_FAIL;
+	}
+
+	ret = test(1, 0, 1, 1);
+	if (ret) {
+		fprintf(stderr, "test 1 0 1 1 failed\n");
+		return T_EXIT_FAIL;
+	}
+
+	ret = test(1, 1, 0, 1);
+	if (ret) {
+		fprintf(stderr, "test 1 1 0 1 failed\n");
+		return T_EXIT_FAIL;
+	}
+
+	ret = test(1, 1, 1, 1);
+	if (ret) {
+		fprintf(stderr, "test 1 1 1 1 failed\n");
+		return T_EXIT_FAIL;
+	}
+
 	ret = test_invalid(0);
 	if (ret) {
 		fprintf(stderr, "test_invalid 0 failed\n");
@@ -400,5 +619,41 @@ int main(int argc, char *argv[])
 		return T_EXIT_FAIL;
 	}
 
+	ret = test_inc(0, 0);
+	if (ret) {
+		fprintf(stderr, "test_inc 0 0 failed\n");
+		return T_EXIT_FAIL;
+	}
+
+	ret = test_inc(0, IORING_SETUP_SQPOLL);
+	if (ret) {
+		fprintf(stderr, "test_inc 0 sqpoll failed\n");
+		return T_EXIT_FAIL;
+	}
+
+	ret = test_inc(0, IORING_SETUP_SINGLE_ISSUER | IORING_SETUP_DEFER_TASKRUN);
+	if (ret) {
+		fprintf(stderr, "test_inc 0 defer failed\n");
+		return T_EXIT_FAIL;
+	}
+
+	ret = test_inc(1, 0);
+	if (ret) {
+		fprintf(stderr, "test_inc 1 0 failed\n");
+		return T_EXIT_FAIL;
+	}
+
+	ret = test_inc(1, IORING_SETUP_SQPOLL);
+	if (ret) {
+		fprintf(stderr, "test_inc 1 sqpoll failed\n");
+		return T_EXIT_FAIL;
+	}
+
+	ret = test_inc(1, IORING_SETUP_SINGLE_ISSUER | IORING_SETUP_DEFER_TASKRUN);
+	if (ret) {
+		fprintf(stderr, "test_inc 1 defer failed\n");
+		return T_EXIT_FAIL;
+	}
+
 	return T_EXIT_PASS;
 }

data/vendor/liburing/test/read-write.c

@@ -15,6 +15,7 @@
 
 #include "helpers.h"
 #include "liburing.h"
+#include "../src/syscall.h"
 
 #define FILE_SIZE	(256 * 1024)
 #define BS		8192
@@ -23,6 +24,7 @@
 static struct iovec *vecs;
 static int no_read;
 static int no_buf_select;
+static int no_buf_copy;
 static int warned;
 
 static int create_nonaligned_buffers(void)
@@ -42,9 +44,9 @@ static int create_nonaligned_buffers(void)
 	return 0;
 }
 
-static int __test_io(const char *file, struct io_uring *ring, int write,
-		     int buffered, int sqthread, int fixed, int nonvec,
-		     int buf_select, int seq, int exp_len)
+static int _test_io(const char *file, struct io_uring *ring, int write,
+		    int buffered, int sqthread, int fixed, int nonvec,
+		    int buf_select, int seq, int exp_len)
 {
 	struct io_uring_sqe *sqe;
 	struct io_uring_cqe *cqe;
@@ -64,7 +66,7 @@ static int __test_io(const char *file, struct io_uring *ring, int write,
 	if (!buffered)
 		open_flags |= O_DIRECT;
 
-	if (fixed) {
+	if (fixed == 1) {
 		ret = t_register_buffers(ring, vecs, BUFFERS);
 		if (ret == T_SETUP_SKIP)
 			return 0;
@@ -76,7 +78,7 @@ static int __test_io(const char *file, struct io_uring *ring, int write,
 
 	fd = open(file, open_flags);
 	if (fd < 0) {
-		if (errno == EINVAL)
+		if (errno == EINVAL || errno == EPERM || errno == EACCES)
 			return 0;
 		perror("file open");
 		goto err;
@@ -201,13 +203,6 @@ static int __test_io(const char *file, struct io_uring *ring, int write,
 		io_uring_cqe_seen(ring, cqe);
 	}
 
-	if (fixed) {
-		ret = io_uring_unregister_buffers(ring);
-		if (ret) {
-			fprintf(stderr, "buffer unreg failed: %d\n", ret);
-			goto err;
-		}
-	}
 	if (sqthread) {
 		ret = io_uring_unregister_files(ring);
 		if (ret) {
@@ -229,6 +224,65 @@ err:
 		close(fd);
 	return 1;
 }
+
+static int __test_io(const char *file, struct io_uring *ring, int write,
+		     int buffered, int sqthread, int fixed, int nonvec,
+		     int buf_select, int seq, int exp_len)
+{
+	int ret;
+
+	ret = _test_io(file, ring, write, buffered, sqthread, fixed, nonvec,
+		       buf_select, seq, exp_len);
+	if (ret)
+		return ret;
+
+	if (fixed) {
+		struct io_uring ring2;
+		int ring_flags = 0;
+
+		if (no_buf_copy)
+			return 0;
+		if (sqthread)
+			ring_flags = IORING_SETUP_SQPOLL;
+		ret = t_create_ring(64, &ring2, ring_flags);
+		if (ret == T_SETUP_SKIP)
+			return 0;
+		if (ret != T_SETUP_OK) {
+			fprintf(stderr, "ring create failed: %d\n", ret);
+			return 1;
+		}
+
+		ret = io_uring_clone_buffers(&ring2, ring);
+		if (ret) {
+			if (ret == -EINVAL) {
+				no_buf_copy = 1;
+				io_uring_queue_exit(&ring2);
+				return 0;
+			}
+			fprintf(stderr, "copy buffers: %d\n", ret);
+			return ret;
+		}
+		ret = _test_io(file, &ring2, write, buffered, sqthread, 2,
+			       nonvec, buf_select, seq, exp_len);
+		if (ret)
+			return ret;
+
+		ret = io_uring_unregister_buffers(ring);
+		if (ret) {
+			fprintf(stderr, "buffer unreg failed: %d\n", ret);
+			return ret;
+		}
+		ret = io_uring_unregister_buffers(&ring2);
+		if (ret) {
+			fprintf(stderr, "buffer copy unreg failed: %d\n", ret);
+			return ret;
+		}
+		io_uring_queue_exit(&ring2);
+	}
+
+	return ret;
+}
+
 static int test_io(const char *file, int write, int buffered, int sqthread,
 		   int fixed, int nonvec, int exp_len)
 {
@@ -266,6 +320,8 @@ static int read_poll_link(const char *file)
 
 	fd = open(file, O_WRONLY);
 	if (fd < 0) {
+		if (errno == EACCES || errno == EPERM)
+			return T_EXIT_SKIP;
 		perror("open");
 		return 1;
 	}
@@ -338,6 +394,7 @@ out:
 	if (!(p->ops[IORING_OP_READ].flags & IO_URING_OP_SUPPORTED))
 		goto out;
 	io_uring_queue_exit(&ring);
+	free(p);
 	return 1;
 }
 
@@ -696,6 +753,8 @@ static int test_io_link(const char *file)
 
 	fd = open(file, O_WRONLY);
 	if (fd < 0) {
+		if (errno == EPERM || errno == EACCES)
+			return 0;
 		perror("file open");
 		goto err;
 	}
@@ -929,7 +988,7 @@ int main(int argc, char *argv[])
 	}
 
 	ret = read_poll_link(fname);
-	if (ret) {
+	if (ret == T_EXIT_FAIL) {
 		fprintf(stderr, "read_poll_link failed\n");
 		goto err;
 	}
@@ -970,6 +1029,12 @@ int main(int argc, char *argv[])
 		goto err;
 	}
 
+	if(vecs != NULL) {
+		for (i = 0; i < BUFFERS; i++)
+			free(vecs[i].iov_base);
+	}
+	free(vecs);
+
 	srand((unsigned)time(NULL));
 	if (create_nonaligned_buffers()) {
 		fprintf(stderr, "file creation failed\n");

data/vendor/liburing/test/recv-msgall-stream.c

@@ -317,6 +317,7 @@ static int do_send(struct recv_data *rd)
 		if (cqe->res == -EINVAL) {
 			fprintf(stdout, "send not supported, skipping\n");
 			close(sockfd);
+			free(buf);
 			return 0;
 		}
 		if (cqe->res != iov.iov_len) {
@@ -328,10 +329,12 @@ static int do_send(struct recv_data *rd)
 
 	shutdown(sockfd, SHUT_RDWR);
 	close(sockfd);
+	free(buf);
 	return 0;
 err:
 	shutdown(sockfd, SHUT_RDWR);
 	close(sockfd);
+	free(buf);
 	return 1;
 }
 

data/vendor/liburing/test/recv-msgall.c

@@ -171,12 +171,14 @@ static int do_send(void)
 	sockfd = socket(AF_INET, SOCK_DGRAM, 0);
 	if (sockfd < 0) {
 		perror("socket");
+		free(buf);
 		return 1;
 	}
 
 	ret = connect(sockfd, (struct sockaddr *)&saddr, sizeof(saddr));
 	if (ret < 0) {
 		perror("connect");
+		free(buf);
 		return 1;
 	}
 
@@ -201,6 +203,7 @@ static int do_send(void)
 		if (cqe->res == -EINVAL) {
 			fprintf(stdout, "send not supported, skipping\n");
 			close(sockfd);
+			free(buf);
 			return 0;
 		}
 		if (cqe->res != iov.iov_len) {
@@ -211,9 +214,11 @@ static int do_send(void)
 	}
 
 	close(sockfd);
+	free(buf);
 	return 0;
 err:
 	close(sockfd);
+	free(buf);
 	return 1;
 }