uringmachine 0.24.0 → 0.26.0

data/vendor/liburing/src/include/liburing.h

@@ -138,7 +138,8 @@ struct io_uring_sq {
 	unsigned ring_mask;
 	unsigned ring_entries;
 
-	unsigned pad[2];
+	unsigned sqes_sz;
+	unsigned pad;
 };
 
 struct io_uring_cq {
@@ -947,7 +948,7 @@ IOURINGINLINE void io_uring_prep_epoll_wait(struct io_uring_sqe *sqe, int fd,
 }
 
 IOURINGINLINE void io_uring_prep_files_update(struct io_uring_sqe *sqe,
-					      const int *fds, unsigned nr_fds,
+					      int *fds, unsigned nr_fds,
 					      int offset)
 	LIBURING_NOEXCEPT
 {

data/vendor/liburing/src/liburing-ffi.map

@@ -260,3 +260,7 @@ LIBURING_2.13 {
 		io_uring_get_sqe128;
 		io_uring_prep_cmd_getsockname;
 } LIBURING_2.12;
+
+LIBURING_2.14 {
+
+} LIBURING_2.13;

data/vendor/liburing/src/liburing.map

@@ -134,3 +134,7 @@ LIBURING_2.12 {
 LIBURING_2.13 {
 
 } LIBURING_2.12;
+
+LIBURING_2.14 {
+
+} LIBURING_2.13;

data/vendor/liburing/src/queue.c

@@ -205,6 +205,18 @@ static unsigned __io_uring_flush_sq(struct io_uring *ring)
 	struct io_uring_sq *sq = &ring->sq;
 	unsigned tail = sq->sqe_tail;
 
+	/*
+	 * With IORING_SETUP_SQ_REWIND, the kernel ignores the SQ head / tail
+	 * and submits entries from the beginning of the queue. Continue using
+	 * the indices as before but reset the tail on submission. With the
+	 * head kept to be zero, io_uring_get_sqe() / etc. will work without
+	 * any extra changes.
+	 */
+	if (ring->flags & IORING_SETUP_SQ_REWIND) {
+		sq->sqe_tail = 0;
+		return tail;
+	}
+
 	if (sq->sqe_head != tail) {
 		sq->sqe_head = tail;
 		/*

data/vendor/liburing/src/register.c

@@ -465,6 +465,7 @@ int io_uring_resize_rings(struct io_uring *ring, struct io_uring_params *p)
 
 	sq_head = ring->sq.sqe_head;
 	sq_tail = ring->sq.sqe_tail;
+	__sys_munmap(ring->sq.sqes, ring->sq.sqes_sz);
 	io_uring_unmap_rings(&ring->sq, &ring->cq);
 	memset(&ring->sq, 0, sizeof(ring->sq));
 	memset(&ring->cq, 0, sizeof(ring->cq));

data/vendor/liburing/src/setup.c

@@ -110,6 +110,7 @@ static size_t params_cq_size(const struct io_uring_params *p, unsigned cqes)
 int io_uring_mmap(int fd, struct io_uring_params *p, struct io_uring_sq *sq,
 		  struct io_uring_cq *cq)
 {
+	size_t sqes_sz;
 	int ret;
 
 	sq->ring_sz = p->sq_off.array + p->sq_entries * sizeof(unsigned);
@@ -139,7 +140,14 @@ int io_uring_mmap(int fd, struct io_uring_params *p, struct io_uring_sq *sq,
 		}
 	}
 
-	sq->sqes = __sys_mmap(0, params_sqes_size(p, p->sq_entries),
+	sqes_sz = params_sqes_size(p, p->sq_entries);
+	sq->sqes_sz = (unsigned int) sqes_sz;
+	if (sq->sqes_sz != sqes_sz) {
+		ret = -EINVAL;
+		goto err;
+	}
+
+	sq->sqes = __sys_mmap(0, sq->sqes_sz,
 			      PROT_READ | PROT_WRITE, MAP_SHARED | MAP_POPULATE,
 			      fd, IORING_OFF_SQES);
 	if (IS_ERR(sq->sqes)) {
@@ -227,15 +235,14 @@ static int io_uring_alloc_huge(unsigned entries, struct io_uring_params *p,
 	if (ret)
 		return ret;
 
-	ring_mem = KRING_SIZE;
-
 	sqes_mem = params_sqes_size(p, sq_entries);
 	if (!(p->flags & IORING_SETUP_NO_SQARRAY))
 		sqes_mem += sq_entries * sizeof(unsigned);
 	sqes_mem = (sqes_mem + page_size - 1) & ~(page_size - 1);
 
-	ring_mem += sqes_mem + params_cq_size(p, cq_entries);
-	mem_used = ring_mem;
+	ring_mem = KRING_SIZE;
+	ring_mem += params_cq_size(p, cq_entries);
+	mem_used = sqes_mem + ring_mem;
 	mem_used = (mem_used + page_size - 1) & ~(page_size - 1);
 
 	/*
@@ -269,6 +276,7 @@ static int io_uring_alloc_huge(unsigned entries, struct io_uring_params *p,
 	}
 
 	sq->sqes = ptr;
+	sq->sqes_sz = (unsigned int) sqes_size;
 	if (mem_used <= buf_size) {
 		sq->ring_ptr = (void *) sq->sqes + sqes_mem;
 		/* clear ring sizes, we have just one mmap() to undo */
@@ -332,7 +340,7 @@ int __io_uring_queue_init_params(unsigned entries, struct io_uring *ring,
 	if (fd < 0) {
 		if ((p->flags & IORING_SETUP_NO_MMAP) &&
 		    !(ring->int_flags & INT_FLAG_APP_MEM)) {
-			__sys_munmap(ring->sq.sqes, ret);
+			__sys_munmap(ring->sq.sqes, ring->sq.sqes_sz);
 			io_uring_unmap_rings(&ring->sq, &ring->cq);
 		}
 		return fd;
@@ -447,7 +455,7 @@ __cold void io_uring_queue_exit(struct io_uring *ring)
 	struct io_uring_cq *cq = &ring->cq;
 
 	if (!(ring->int_flags & INT_FLAG_APP_MEM)) {
-		__sys_munmap(sq->sqes, io_uring_sqes_size(ring));
+		__sys_munmap(sq->sqes, sq->sqes_sz);
 		io_uring_unmap_rings(sq, cq);
 	}
 

data/vendor/liburing/test/Makefile

@@ -132,6 +132,7 @@ test_srcs := \
 	io_uring_passthrough.c \
 	io_uring_register.c \
 	io_uring_setup.c \
+	io-wq-exit.c \
 	iowait.c \
 	kallsyms.c \
 	lfs-openat.c \
@@ -144,6 +145,7 @@ test_srcs := \
 	min-timeout.c \
 	min-timeout-wait.c \
 	mkdir.c \
+	mock_file.c \
 	msg-ring.c \
 	msg-ring-fd.c \
 	msg-ring-flags.c \
@@ -176,6 +178,7 @@ test_srcs := \
 	poll-race.c \
 	poll-race-mshot.c \
 	poll-ring.c \
+	poll-update-trigger.c \
 	poll-v-poll.c \
 	pollfree.c \
 	probe.c \
@@ -205,6 +208,7 @@ test_srcs := \
 	ringbuf-status.c \
 	ring-leak2.c \
 	ring-leak.c \
+	ring-query.c \
 	rsrc_tags.c \
 	rw_merge_test.c \
 	self.c \
@@ -214,6 +218,7 @@ test_srcs := \
 	send_recvmsg.c \
 	send-zerocopy.c \
 	sendmsg_iov_clean.c \
+	sendzc-bug.c \
 	shared-wq.c \
 	short-read.c \
 	shutdown.c \
@@ -252,10 +257,13 @@ test_srcs := \
 	timerfd-short-read.c \
 	timeout.c \
 	timeout-new.c \
+	timestamp.c \
+	timestamp-bug.c \
 	truncate.c \
 	tty-write-dpoll.c \
 	unlink.c \
 	uring_cmd_ublk.c \
+	vec-regbuf.c \
 	version.c \
 	waitid.c \
 	wait-timeout.c \
@@ -263,10 +271,6 @@ test_srcs := \
 	wq-aff.c \
 	xattr.c \
 	zcrx.c \
-	vec-regbuf.c \
-	timestamp.c \
-	ring-query.c \
-	mock_file.c \
 	# EOL
 
 # Please keep this list sorted alphabetically.

data/vendor/liburing/test/conn-unreach.c

@@ -30,7 +30,7 @@ static int check_cqe(struct io_uring *ring, struct io_uring_cqe *cqe)
 
 	switch (cqe->user_data) {
 	case 1:
-		if (cqe->res != -ECONNRESET) {
+		if (cqe->res != -ECONNRESET && cqe->res != -ENETUNREACH) {
 			fprintf(stderr, "Unexpected connect: %d\n", cqe->res);
 			return T_EXIT_FAIL;
 		}

data/vendor/liburing/test/epwait.c

@@ -10,11 +10,13 @@
 #include <poll.h>
 #include <string.h>
 #include <pthread.h>
+#include <stdatomic.h>
 #include "liburing.h"
 #include "helpers.h"
 
 static int fds[2][2];
 static int no_epoll_wait;
+static atomic_bool keep_running;
 
 static int test_ready(struct io_uring *ring, int efd)
 {
@@ -193,8 +195,8 @@ static int test_remove(struct io_uring *ring, int efd)
 	return 0;
 }
 
-#define LOOPS	500
 #define NPIPES	8
+#define MAX_QE 1000
 
 struct d {
 	int pipes[NPIPES][2];
@@ -203,9 +205,9 @@ struct d {
 static void *thread_fn(void *data)
 {
 	struct d *d = data;
-	int i, j;
+	int i;
 
-	for (j = 0; j < LOOPS; j++) {
+	while (atomic_load_explicit(&keep_running, memory_order_relaxed)) {
 		usleep(150);
 		for (i = 0; i < NPIPES; i++) {
 			int ret;
@@ -240,8 +242,12 @@ static int test_race(int flags)
 	struct epoll_event ev;
 	struct epoll_event out[NPIPES];
 	pthread_t thread;
-	int i, j, efd, ret;
+	int i, efd, ret;
 	void *tret;
+	int sqe_id = 0;
+	int submitted = 0, completed = 0;
+	int sqe_ids[MAX_QE];
+	int cqe_ids[MAX_QE];
 
 	ret = t_create_ring(32, &ring, flags);
 	if (ret == T_SETUP_SKIP) {
@@ -278,16 +284,19 @@ static int test_race(int flags)
 	io_uring_prep_epoll_wait(sqe, efd, out, NPIPES, 0);
 	io_uring_submit(&ring);
 
+	atomic_store(&keep_running, true);
 	pthread_create(&thread, NULL, thread_fn, &d);
 
-	for (j = 0; j < LOOPS; j++) {
+	while (completed < 1000) {
 		io_uring_submit_and_wait(&ring, 1);
-
+		sqe_ids[submitted] = sqe->user_data;
 		ret = io_uring_wait_cqe(&ring, &cqe);
 		if (ret) {
 			fprintf(stderr, "wait %d\n", ret);
 			return 1;
 		}
+		cqe_ids[completed] = cqe->user_data;
+		completed++;
 		if (cqe->res < 0) {
 			fprintf(stderr, "race res %d\n", cqe->res);
 			return 1;
@@ -297,10 +306,27 @@ static int test_race(int flags)
 		usleep(100);
 		sqe = io_uring_get_sqe(&ring);
 		io_uring_prep_epoll_wait(sqe, efd, out, NPIPES, 0);
+		sqe->user_data = ++sqe_id;
+		submitted++;
 	}
 
+	atomic_store(&keep_running, false);
 	pthread_join(thread, &tret);
 
+	if (submitted != completed) {
+		fprintf(stderr, "SQE/CQE mismatch: submitted=%d completed=%d\n",
+			submitted, completed);
+		return 1;
+	}
+
+	for (i = 0; i < completed; i++) {
+		if (sqe_ids[i] != cqe_ids[i]) {
+			fprintf(stderr, "user_data mismatch at %d: sqe=%d cqe=%d\n",
+				i, sqe_ids[i], cqe_ids[i]);
+			return 1;
+		}
+	}
+
 	for (i = 0; i < NPIPES; i++) {
 		close(d.pipes[i][0]);
 		close(d.pipes[i][1]);

data/vendor/liburing/test/io-wq-exit.c

@@ -0,0 +1,131 @@
+/* SPDX-License-Identifier: MIT */
+/*
+ * Description: Test that created thread that sets up a ring and causes io-wq
+ *		worker creation doesn't need to wait for io-wq idle worker
+ *		timeout to exit.
+ *
+ */
+#include <fcntl.h>
+#include <pthread.h>
+#include <stddef.h>
+#include <stdint.h>
+#include <stdio.h>
+#include <string.h>
+#include <time.h>
+#include <unistd.h>
+
+#include "liburing.h"
+#include "helpers.h"
+
+#define BUF_LEN		20
+
+static void *test(void *data)
+{
+	int fd_src = -1, fd_dst = -1, pipe_fds[2] = { -1, -1 };
+	uint8_t buffer_write[BUF_LEN], buffer_read[BUF_LEN];
+	char dst_fname[PATH_MAX], src_fname[PATH_MAX];
+	struct io_uring_sqe *sqe;
+	struct io_uring_cqe *cqe;
+	struct io_uring ring;
+	int ret;
+
+	ret = io_uring_queue_init(4, &ring, 0);
+	if (ret < 0) {
+		fprintf(stderr, "io_uring_queue_init failed: %d\n", ret);
+		return NULL;
+	}
+
+	sprintf(src_fname, ".splice.%d.src", getpid());
+	fd_src = open(src_fname, O_RDWR | O_CREAT | O_TRUNC, 0644);
+	if (fd_src < 0) {
+		perror("open source");
+		goto cleanup;
+	}
+
+	sprintf(dst_fname, ".splice.%d.dst", getpid());
+	fd_dst = open(dst_fname, O_RDWR | O_CREAT | O_TRUNC, 0644);
+	if (fd_dst < 0) {
+		perror("open destination");
+		goto cleanup;
+	}
+
+	memset(buffer_write, 97, BUF_LEN);
+	memset(buffer_read, 98, BUF_LEN);
+	ret = pwrite(fd_src, buffer_write, BUF_LEN, 0);
+	if (ret < 0) {
+		perror("pwrite src");
+		goto cleanup;
+	}
+
+	if (pipe(pipe_fds) < 0) {
+		perror("pipe");
+		goto cleanup;
+	}
+
+	sqe = io_uring_get_sqe(&ring);
+	io_uring_prep_splice(sqe, fd_src, 0, pipe_fds[1], -1, 20, 0);
+	sqe->user_data = 1;
+
+	sqe = io_uring_get_sqe(&ring);
+	io_uring_prep_splice(sqe, pipe_fds[0], -1, fd_dst, 10, 20, 0);
+	sqe->user_data = 2;
+
+	io_uring_submit(&ring);
+
+	ret = io_uring_wait_cqe(&ring, &cqe);
+	if (ret) {
+		fprintf(stderr, "wait cqe %d\n", ret);
+		goto cleanup;
+	}
+	io_uring_cqe_seen(&ring, cqe);
+
+	ret = io_uring_wait_cqe(&ring, &cqe);
+	if (ret) {
+		fprintf(stderr, "wait cqe %d\n", ret);
+		goto cleanup;
+	}
+	io_uring_cqe_seen(&ring, cqe);
+cleanup:
+	if (pipe_fds[0] != -1)
+		close(pipe_fds[0]);
+	if (pipe_fds[1] != -1)
+		close(pipe_fds[1]);
+	if (fd_src != -1)
+		close(fd_src);
+	if (fd_src != -1)
+		close(fd_dst);
+	io_uring_queue_exit(&ring);
+	unlink(src_fname);
+	unlink(dst_fname);
+	return NULL;
+}
+
+static long get_time_ns(void)
+{
+	struct timespec ts;
+
+	clock_gettime(CLOCK_MONOTONIC, &ts);
+	return ts.tv_sec * 1000000000L + ts.tv_nsec;
+}
+
+int main(int argc, char *argv[])
+{
+	pthread_t thread;
+	long start, end;
+
+	if (argc > 1)
+		return T_EXIT_SKIP;
+
+	start = get_time_ns();
+	pthread_create(&thread, NULL, test, NULL);
+	pthread_join(thread, NULL);
+	end = get_time_ns();
+	end -= start;
+	end /= 1000000;
+	if (end >= 500) {
+		fprintf(stderr, "Test took too long: %lu msec\n", end);
+		return T_EXIT_FAIL;
+	}
+
+	return T_EXIT_PASS;
+}

data/vendor/liburing/test/iowait.c

@@ -90,7 +90,7 @@ static int test(struct io_uring *ring, int with_iowait, int cpu)
 	io_uring_cqe_seen(ring, cqe);
 
 	if (with_iowait) {
-		if (diff < 90) {
+		if (diff < 50) {
 			fprintf(stderr, "iowait diff too small: %d\n", diff);
 			return T_EXIT_FAIL;
 		}

data/vendor/liburing/test/min-timeout.c

@@ -133,8 +133,10 @@ static int test(int flags, int expected_ctx, int min_wait, int write_delay,
 		io_uring_cqe_seen(&ring, cqe);
 	}
 
-	if (i != nr_cqes)
+	if (i != nr_cqes) {
 		fprintf(stderr, "Got %d CQEs, expected %d\n", i, nr_cqes);
+		return T_EXIT_FAIL;
+	}
 
 	pthread_join(thread, &tret);
 

data/vendor/liburing/test/open-close.c

@@ -217,6 +217,39 @@ err:
 	return -1;
 }
 
+static int test_open_direct_cloexec(struct io_uring *ring, const char *path, int dfd)
+{
+	struct io_uring_cqe *cqe;
+	struct io_uring_sqe *sqe;
+	int ret;
+
+	sqe = io_uring_get_sqe(ring);
+	if (!sqe) {
+		fprintf(stderr, "get sqe failed\n");
+		goto err;
+	}
+	io_uring_prep_openat(sqe, dfd, path, O_RDONLY | O_CLOEXEC, 0);
+	sqe->file_index = 1;
+
+	ret = io_uring_submit(ring);
+	if (ret <= 0) {
+		fprintf(stderr, "sqe submit failed: %d\n", ret);
+		goto err;
+	}
+
+	ret = io_uring_wait_cqe(ring, &cqe);
+	if (ret < 0) {
+		fprintf(stderr, "wait completion %d\n", ret);
+		goto err;
+	}
+	if (cqe->res != -EINVAL)
+		return 1;
+	io_uring_cqe_seen(ring, cqe);
+	return 0;
+err:
+	return 1;
+}
+
 int main(int argc, char *argv[])
 {
 	struct io_uring ring;
@@ -288,6 +321,12 @@ int main(int argc, char *argv[])
 		goto err;
 	}
 
+	ret = test_open_direct_cloexec(&ring, path, -1);
+	if (ret) {
+		fprintf(stderr, "test_open_direct_cloexex failed\n");
+		goto err;
+	}
+
 done:
 	unlink(path);
 	if (do_unlink)

data/vendor/liburing/test/poll-update-trigger.c

@@ -0,0 +1,85 @@
+/* SPDX-License-Identifier: MIT */
+/*
+ * Description: test POLL_ADD being added with a mask that doesn't make it
+ *		trigger, but then updated with POLL_REMOVE to a mask that does
+ *		trigger it.
+ */
+#include <stdio.h>
+#include <stdlib.h>
+#include <unistd.h>
+#include <fcntl.h>
+#include <sys/stat.h>
+#include <sys/types.h>
+#include <poll.h>
+
+#include "liburing.h"
+#include "helpers.h"
+
+int main(int argc, char *argv[])
+{
+	struct io_uring_sqe *sqe;
+	struct io_uring_cqe *cqe;
+	struct io_uring ring;
+	int ret, fd[2], i;
+
+	if (argc > 1)
+		return T_EXIT_SKIP;
+
+	ret = io_uring_queue_init(4, &ring, 0);
+	if (ret) {
+		fprintf(stderr, "queue_init: %d\n", ret);
+		return T_EXIT_FAIL;
+	}
+
+	if (pipe(fd) < 0) {
+		perror("pipe");
+		return T_EXIT_FAIL;
+	}
+
+	/*
+	 * Attach POLLIN request to output side of pipe, won't complete as
+	 * that will never be readabable.
+	 */
+	sqe = io_uring_get_sqe(&ring);
+	io_uring_prep_poll_add(sqe, fd[1], POLLIN);
+	sqe->user_data = 1;
+	io_uring_submit(&ring);
+
+	/*
+	 * Now update the previously submitted poll request, adding POLLOUT.
+	 * This will cause it to trigger immediately, as the pipe is indeed
+	 * writeable.
+	 */
+	sqe = io_uring_get_sqe(&ring);
+	io_uring_prep_poll_remove(sqe, 1);
+	sqe->len = IORING_POLL_UPDATE_EVENTS;
+	sqe->poll32_events = POLLOUT;
+	sqe->user_data = 2;
+	io_uring_submit(&ring);
+
+	/*
+	 * Expect both requests to complete - the update with 0 for success,
+	 * and the original poll_add with POLLOUT as the result.
+	 */
+	for (i = 0; i < 2; i++) {
+		ret = io_uring_wait_cqe(&ring, &cqe);
+		if (ret) {
+			fprintf(stderr, "wait_cqe: %d\n", ret);
+			return T_EXIT_FAIL;
+		}
+		if (cqe->user_data == 1) {
+			if (cqe->res != POLLOUT) {
+				fprintf(stderr, "Original poll: %d\n", cqe->res);
+				return T_EXIT_FAIL;
+			}
+		} else if (cqe->user_data == 2) {
+			if (cqe->res) {
+				fprintf(stderr, "Poll update: %d\n", cqe->res);
+				return T_EXIT_FAIL;
+			}
+		}
+		io_uring_cqe_seen(&ring, cqe);
+	}
+
+	return T_EXIT_PASS;
+}

data/vendor/liburing/test/recvsend_bundle.c

@@ -333,12 +333,9 @@ err:
 	return (void *)(intptr_t)ret;
 }
 
-static int __do_send_bundle(struct recv_data *rd, struct io_uring *ring, int sockfd)
+static void submit_bundle(struct io_uring *ring, int sockfd)
 {
-	struct io_uring_cqe *cqe;
 	struct io_uring_sqe *sqe;
-	int bytes_needed = MSG_SIZE * nr_msgs;
-	int i, ret;
 
 	sqe = io_uring_get_sqe(ring);
 	io_uring_prep_send_bundle(sqe, sockfd, 0, 0);
@@ -346,9 +343,16 @@ static int __do_send_bundle(struct recv_data *rd, struct io_uring *ring, int soc
 	sqe->buf_group = SEND_BGID;
 	sqe->user_data = 1;
 
-	ret = io_uring_submit(ring);
-	if (ret != 1)
-		return 1;
+	io_uring_submit(ring);
+}
+
+static int __do_send_bundle(struct recv_data *rd, struct io_uring *ring, int sockfd)
+{
+	struct io_uring_cqe *cqe;
+	int bytes_needed = MSG_SIZE * nr_msgs;
+	int i, ret;
+
+	submit_bundle(ring, sockfd);
 
 	pthread_barrier_wait(&rd->barrier);
 
@@ -372,10 +376,9 @@ static int __do_send_bundle(struct recv_data *rd, struct io_uring *ring, int soc
 			io_uring_cqe_seen(ring, cqe);
 			break;
 		}
-		if (!(cqe->flags & IORING_CQE_F_MORE)) {
-			fprintf(stderr, "expected more, but MORE not set\n");
-			return 1;
-		}
+		/* bundle finished, resubmit as more is pending */
+		if (!(cqe->flags & IORING_CQE_F_MORE))
+			submit_bundle(ring, sockfd);
 		io_uring_cqe_seen(ring, cqe);
 	}