polyphony 0.94 → 0.95

data/vendor/liburing/src/int_flags.h

@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: MIT */
 #ifndef LIBURING_INT_FLAGS
 #define LIBURING_INT_FLAGS
 

data/vendor/liburing/src/lib.h

@@ -6,33 +6,37 @@
 #include <string.h>
 #include <unistd.h>
 
-#define __INTERNAL__LIBURING_LIB_H
 #if defined(__x86_64__) || defined(__i386__)
-	#include "arch/x86/lib.h"
+#include "arch/x86/lib.h"
+#elif defined(__aarch64__)
+#include "arch/aarch64/lib.h"
 #else
-	/*
-	 * We don't have nolibc support for this arch. Must use libc!
-	 */
-	#ifdef CONFIG_NOLIBC
-		#error "This arch doesn't support building liburing without libc"
-	#endif
-	/* libc wrappers. */
-	#include "arch/generic/lib.h"
+/*
+ * We don't have nolibc support for this arch. Must use libc!
+ */
+#ifdef CONFIG_NOLIBC
+#error "This arch doesn't support building liburing without libc"
+#endif
+/* libc wrappers. */
+#include "arch/generic/lib.h"
 #endif
-#undef __INTERNAL__LIBURING_LIB_H
 
 
 #ifndef offsetof
-	#define offsetof(TYPE, FIELD) ((size_t) &((TYPE *)0)->FIELD)
+#define offsetof(TYPE, FIELD) ((size_t) &((TYPE *)0)->FIELD)
 #endif
 
 #ifndef container_of
-	#define container_of(PTR, TYPE, FIELD) ({			\
-		__typeof__(((TYPE *)0)->FIELD) *__FIELD_PTR = (PTR);	\
-		(TYPE *)((char *) __FIELD_PTR - offsetof(TYPE, FIELD));	\
-	})
+#define container_of(PTR, TYPE, FIELD) ({			\
+	__typeof__(((TYPE *)0)->FIELD) *__FIELD_PTR = (PTR);	\
+	(TYPE *)((char *) __FIELD_PTR - offsetof(TYPE, FIELD));	\
+})
 #endif
 
+#define __maybe_unused		__attribute__((__unused__))
+#define __hot			__attribute__((__hot__))
+#define __cold			__attribute__((__cold__))
+
 void *__uring_malloc(size_t len);
 void __uring_free(void *p);
 

data/vendor/liburing/src/liburing.map

@@ -50,4 +50,20 @@ LIBURING_2.2 {
 		io_uring_submit_and_wait_timeout;
 		io_uring_register_ring_fd;
 		io_uring_unregister_ring_fd;
+		io_uring_register_files_sparse;
+		io_uring_register_buffers_sparse;
+		io_uring_register_buf_ring;
+		io_uring_unregister_buf_ring;
 } LIBURING_2.1;
+
+LIBURING_2.3 {
+	global:
+		io_uring_register_sync_cancel;
+		io_uring_register_file_alloc_range;
+		io_uring_enter;
+		io_uring_enter2;
+		io_uring_setup;
+		io_uring_register;
+		io_uring_get_events;
+		io_uring_submit_and_get_events;
+} LIBURING_2.2;

data/vendor/liburing/src/nolibc.c

@@ -1,7 +1,7 @@
 /* SPDX-License-Identifier: MIT */
 
 #ifndef CONFIG_NOLIBC
-# error "This file should only be compiled for no libc build"
+#error "This file should only be compiled for no libc build"
 #endif
 
 #include "lib.h"

data/vendor/liburing/src/queue.c

@@ -12,9 +12,15 @@
  * Returns true if we're not using SQ thread (thus nobody submits but us)
  * or if IORING_SQ_NEED_WAKEUP is set, so submit thread must be explicitly
  * awakened. For the latter case, we set the thread wakeup flag.
+ * If no SQEs are ready for submission, returns false.
  */
-static inline bool sq_ring_needs_enter(struct io_uring *ring, unsigned *flags)
+static inline bool sq_ring_needs_enter(struct io_uring *ring,
+				       unsigned submit,
+				       unsigned *flags)
 {
+	if (!submit)
+		return false;
+
 	if (!(ring->flags & IORING_SETUP_SQPOLL))
 		return true;
 
@@ -35,7 +41,8 @@ static inline bool sq_ring_needs_enter(struct io_uring *ring, unsigned *flags)
 
 static inline bool cq_ring_needs_flush(struct io_uring *ring)
 {
-	return IO_URING_READ_ONCE(*ring->sq.kflags) & IORING_SQ_CQ_OVERFLOW;
+	return IO_URING_READ_ONCE(*ring->sq.kflags) &
+				 (IORING_SQ_CQ_OVERFLOW | IORING_SQ_TASKRUN);
 }
 
 static inline bool cq_ring_needs_enter(struct io_uring *ring)
@@ -48,15 +55,17 @@ struct get_data {
 	unsigned wait_nr;
 	unsigned get_flags;
 	int sz;
+	int has_ts;
 	void *arg;
 };
 
-static int _io_uring_get_cqe(struct io_uring *ring, struct io_uring_cqe **cqe_ptr,
+static int _io_uring_get_cqe(struct io_uring *ring,
+			     struct io_uring_cqe **cqe_ptr,
 			     struct get_data *data)
 {
 	struct io_uring_cqe *cqe = NULL;
 	bool looped = false;
-	int err;
+	int err = 0;
 
 	do {
 		bool need_enter = false;
@@ -64,9 +73,12 @@ static int _io_uring_get_cqe(struct io_uring *ring, struct io_uring_cqe **cqe_pt
 		unsigned nr_available;
 		int ret;
 
-		err = __io_uring_peek_cqe(ring, &cqe, &nr_available);
-		if (err)
+		ret = __io_uring_peek_cqe(ring, &cqe, &nr_available);
+		if (ret) {
+			if (!err)
+				err = ret;
 			break;
+		}
 		if (!cqe && !data->wait_nr && !data->submit) {
 			/*
 			 * If we already looped once, we already entererd
@@ -74,7 +86,8 @@ static int _io_uring_get_cqe(struct io_uring *ring, struct io_uring_cqe **cqe_pt
 			 * wait for, don't keep retrying.
 			 */
 			if (looped || !cq_ring_needs_enter(ring)) {
-				err = -EAGAIN;
+				if (!err)
+					err = -EAGAIN;
 				break;
 			}
 			need_enter = true;
@@ -83,25 +96,36 @@ static int _io_uring_get_cqe(struct io_uring *ring, struct io_uring_cqe **cqe_pt
 			flags = IORING_ENTER_GETEVENTS | data->get_flags;
 			need_enter = true;
 		}
-		if (data->submit && sq_ring_needs_enter(ring, &flags))
+		if (sq_ring_needs_enter(ring, data->submit, &flags))
 			need_enter = true;
 		if (!need_enter)
 			break;
+		if (looped && data->has_ts) {
+			struct io_uring_getevents_arg *arg = data->arg;
+
+			if (!cqe && arg->ts && !err)
+				err = -ETIME;
+			break;
+		}
 
 		if (ring->int_flags & INT_FLAG_REG_RING)
 			flags |= IORING_ENTER_REGISTERED_RING;
-		ret = ____sys_io_uring_enter2(ring->enter_ring_fd, data->submit,
-					      data->wait_nr, flags, data->arg,
-					      data->sz);
+		ret = __sys_io_uring_enter2(ring->enter_ring_fd, data->submit,
+					    data->wait_nr, flags, data->arg,
+					    data->sz);
 		if (ret < 0) {
-			err = ret;
+			if (!err)
+				err = ret;
 			break;
 		}
 
 		data->submit -= ret;
 		if (cqe)
 			break;
-		looped = true;
+		if (!looped) {
+			looped = true;
+			err = ret;
+		}
 	} while (1);
 
 	*cqe_ptr = cqe;
@@ -122,6 +146,15 @@ int __io_uring_get_cqe(struct io_uring *ring, struct io_uring_cqe **cqe_ptr,
 	return _io_uring_get_cqe(ring, cqe_ptr, &data);
 }
 
+int io_uring_get_events(struct io_uring *ring)
+{
+	int flags = IORING_ENTER_GETEVENTS;
+
+	if (ring->int_flags & INT_FLAG_REG_RING)
+		flags |= IORING_ENTER_REGISTERED_RING;
+	return __sys_io_uring_enter(ring->enter_ring_fd, 0, 0, flags, NULL);
+}
+
 /*
  * Fill in an array of IO completions up to count, if any are available.
  * Returns the amount of IO completions filled.
@@ -131,37 +164,36 @@ unsigned io_uring_peek_batch_cqe(struct io_uring *ring,
 {
 	unsigned ready;
 	bool overflow_checked = false;
+	int shift = 0;
+
+	if (ring->flags & IORING_SETUP_CQE32)
+		shift = 1;
 
 again:
 	ready = io_uring_cq_ready(ring);
 	if (ready) {
 		unsigned head = *ring->cq.khead;
-		unsigned mask = *ring->cq.kring_mask;
+		unsigned mask = ring->cq.ring_mask;
 		unsigned last;
 		int i = 0;
 
 		count = count > ready ? ready : count;
 		last = head + count;
 		for (;head != last; head++, i++)
-			cqes[i] = &ring->cq.cqes[head & mask];
+			cqes[i] = &ring->cq.cqes[(head & mask) << shift];
 
 		return count;
 	}
 
 	if (overflow_checked)
-		goto done;
+		return 0;
 
 	if (cq_ring_needs_flush(ring)) {
-		int flags = IORING_ENTER_GETEVENTS;
-
-		if (ring->int_flags & INT_FLAG_REG_RING)
-			flags |= IORING_ENTER_REGISTERED_RING;
-		____sys_io_uring_enter(ring->enter_ring_fd, 0, 0, flags, NULL);
+		io_uring_get_events(ring);
 		overflow_checked = true;
 		goto again;
 	}
 
-done:
 	return 0;
 }
 
@@ -169,31 +201,21 @@ done:
  * Sync internal state with kernel ring state on the SQ side. Returns the
  * number of pending items in the SQ ring, for the shared ring.
  */
-int __io_uring_flush_sq(struct io_uring *ring)
+unsigned __io_uring_flush_sq(struct io_uring *ring)
 {
 	struct io_uring_sq *sq = &ring->sq;
-	const unsigned mask = *sq->kring_mask;
-	unsigned ktail = *sq->ktail;
-	unsigned to_submit = sq->sqe_tail - sq->sqe_head;
-
-	if (!to_submit)
-		goto out;
-
-	/*
-	 * Fill in sqes that we have queued up, adding them to the kernel ring
-	 */
-	do {
-		sq->array[ktail & mask] = sq->sqe_head & mask;
-		ktail++;
-		sq->sqe_head++;
-	} while (--to_submit);
-
-	/*
-	 * Ensure that the kernel sees the SQE updates before it sees the tail
-	 * update.
-	 */
-	io_uring_smp_store_release(sq->ktail, ktail);
-out:
+	unsigned tail = sq->sqe_tail;
+
+	if (sq->sqe_head != tail) {
+		sq->sqe_head = tail;
+		/*
+		 * Ensure kernel sees the SQE updates before the tail update.
+		 */
+		if (!(ring->flags & IORING_SETUP_SQPOLL))
+			IO_URING_WRITE_ONCE(*sq->ktail, tail);
+		else
+			io_uring_smp_store_release(sq->ktail, tail);
+	}
 	/*
 	 * This _may_ look problematic, as we're not supposed to be reading
 	 * SQ->head without acquire semantics. When we're in SQPOLL mode, the
@@ -205,7 +227,7 @@ out:
 	 * we can submit. The point is, we need to be able to deal with this
 	 * situation regardless of any perceived atomicity.
 	 */
-	return ktail - *sq->khead;
+	return tail - *sq->khead;
 }
 
 /*
@@ -214,7 +236,8 @@ out:
  */
 static int io_uring_wait_cqes_new(struct io_uring *ring,
 				  struct io_uring_cqe **cqe_ptr,
-				  unsigned wait_nr, struct __kernel_timespec *ts,
+				  unsigned wait_nr,
+				  struct __kernel_timespec *ts,
 				  sigset_t *sigmask)
 {
 	struct io_uring_getevents_arg arg = {
@@ -226,6 +249,7 @@ static int io_uring_wait_cqes_new(struct io_uring *ring,
 		.wait_nr	= wait_nr,
 		.get_flags	= IORING_ENTER_EXT_ARG,
 		.sz		= sizeof(arg),
+		.has_ts		= ts != NULL,
 		.arg		= &arg
 	};
 
@@ -249,7 +273,6 @@ static int io_uring_wait_cqes_new(struct io_uring *ring,
  * hence this function is safe to use for applications that split SQ and CQ
  * handling between two threads.
  */
-
 static int __io_uring_submit_timeout(struct io_uring *ring, unsigned wait_nr,
 				     struct __kernel_timespec *ts)
 {
@@ -311,6 +334,7 @@ int io_uring_submit_and_wait_timeout(struct io_uring *ring,
 				.wait_nr	= wait_nr,
 				.get_flags	= IORING_ENTER_EXT_ARG,
 				.sz		= sizeof(arg),
+				.has_ts		= ts != NULL,
 				.arg		= &arg
 			};
 
@@ -342,20 +366,21 @@ int io_uring_wait_cqe_timeout(struct io_uring *ring,
  * Returns number of sqes submitted
  */
 static int __io_uring_submit(struct io_uring *ring, unsigned submitted,
-			     unsigned wait_nr)
+			     unsigned wait_nr, bool getevents)
 {
+	bool cq_needs_enter = getevents || wait_nr || cq_ring_needs_enter(ring);
 	unsigned flags;
 	int ret;
 
 	flags = 0;
-	if (sq_ring_needs_enter(ring, &flags) || wait_nr) {
-		if (wait_nr || (ring->flags & IORING_SETUP_IOPOLL))
+	if (sq_ring_needs_enter(ring, submitted, &flags) || cq_needs_enter) {
+		if (cq_needs_enter)
 			flags |= IORING_ENTER_GETEVENTS;
 		if (ring->int_flags & INT_FLAG_REG_RING)
 			flags |= IORING_ENTER_REGISTERED_RING;
 
-		ret = ____sys_io_uring_enter(ring->enter_ring_fd, submitted,
-						wait_nr, flags, NULL);
+		ret = __sys_io_uring_enter(ring->enter_ring_fd, submitted,
+					   wait_nr, flags, NULL);
 	} else
 		ret = submitted;
 
@@ -364,7 +389,7 @@ static int __io_uring_submit(struct io_uring *ring, unsigned submitted,
 
 static int __io_uring_submit_and_wait(struct io_uring *ring, unsigned wait_nr)
 {
-	return __io_uring_submit(ring, __io_uring_flush_sq(ring), wait_nr);
+	return __io_uring_submit(ring, __io_uring_flush_sq(ring), wait_nr, false);
 }
 
 /*
@@ -387,10 +412,17 @@ int io_uring_submit_and_wait(struct io_uring *ring, unsigned wait_nr)
 	return __io_uring_submit_and_wait(ring, wait_nr);
 }
 
+int io_uring_submit_and_get_events(struct io_uring *ring)
+{
+	return __io_uring_submit(ring, __io_uring_flush_sq(ring), 0, true);
+}
+
+#ifdef LIBURING_INTERNAL
 struct io_uring_sqe *io_uring_get_sqe(struct io_uring *ring)
 {
 	return _io_uring_get_sqe(ring);
 }
+#endif
 
 int __io_uring_sqring_wait(struct io_uring *ring)
 {
@@ -399,5 +431,5 @@ int __io_uring_sqring_wait(struct io_uring *ring)
 	if (ring->int_flags & INT_FLAG_REG_RING)
 		flags |= IORING_ENTER_REGISTERED_RING;
 
-	return  ____sys_io_uring_enter(ring->enter_ring_fd, 0, 0, flags, NULL);
+	return __sys_io_uring_enter(ring->enter_ring_fd, 0, 0, flags, NULL);
 }