uringmachine 0.4 → 0.5

data/vendor/liburing/configure

@@ -417,6 +417,21 @@ if compile_prog "" "" "futexv"; then
 fi
 print_config "futex waitv support" "$futexv"
 
+##########################################
+# Check block discard cmd support
+discard_cmd="no"
+cat > $TMPC << EOF
+#include <linux/blkdev.h>
+int main(void)
+{
+  return BLOCK_URING_CMD_DISCARD;
+}
+EOF
+if compile_prog "" "" "discard command"; then
+  discard_cmd="yes"
+fi
+print_config "io_uring discard command support" "$discard_cmd"
+
 ##########################################
 # Check idtype_t support
 has_idtype_t="no"
@@ -651,6 +666,23 @@ typedef enum
 } idtype_t;
 EOF
 fi
+
+if test "$discard_cmd" != "yes"; then
+cat >> $compat_h << EOF
+
+#include <linux/ioctl.h>
+
+#ifndef BLOCK_URING_CMD_DISCARD
+#define BLOCK_URING_CMD_DISCARD                        _IO(0x12, 0)
+#endif
+
+EOF
+else cat >> $compat_h << EOF
+#include <linux/blkdev.h>
+
+EOF
+fi
+
 cat >> $compat_h << EOF
 #endif
 EOF

data/vendor/liburing/examples/Makefile

@@ -28,6 +28,7 @@ example_srcs := \
 	napi-busy-poll-client.c \
 	napi-busy-poll-server.c \
 	poll-bench.c \
+	reg-wait.c \
 	send-zerocopy.c \
 	rsrc-update-bench.c \
 	proxy.c \

data/vendor/liburing/examples/reg-wait.c

@@ -0,0 +1,159 @@
+/* SPDX-License-Identifier: MIT */
+/*
+ * Sample program that shows how to use registered waits.
+ *
+ * (C) 2024 Jens Axboe <axboe@kernel.dk>
+ */
+#include <stdint.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <unistd.h>
+#include <assert.h>
+#include <sys/time.h>
+#include <liburing.h>
+
+static unsigned long long mtime_since(const struct timeval *s,
+				      const struct timeval *e)
+{
+	long long sec, usec;
+
+	sec = e->tv_sec - s->tv_sec;
+	usec = (e->tv_usec - s->tv_usec);
+	if (sec > 0 && usec < 0) {
+		sec--;
+		usec += 1000000;
+	}
+
+	sec *= 1000;
+	usec /= 1000;
+	return sec + usec;
+}
+
+static unsigned long long mtime_since_now(struct timeval *tv)
+{
+	struct timeval end;
+
+	gettimeofday(&end, NULL);
+	return mtime_since(tv, &end);
+}
+
+int main(int argc, char *argv[])
+{
+	struct io_uring_reg_wait *reg;
+	struct io_uring_sqe *sqe;
+	struct io_uring_cqe *cqe[2];
+	struct io_uring ring;
+	char b1[8], b2[8];
+	unsigned long msec;
+	struct timeval tv;
+	int ret, fds[2];
+
+	if (argc > 1) {
+		fprintf(stdout, "%s: takes no arguments\n", argv[0]);
+		return 0;
+	}
+
+	if (pipe(fds) < 0) {
+		perror("pipe");
+		return 1;
+	}
+
+	ret = io_uring_queue_init(8, &ring, 0);
+	if (ret) {
+		fprintf(stderr, "Queue init: %d\n", ret);
+		return 1;
+	}
+
+	/*
+	 * Setup wait region. We'll use 32 here, but 64 is probably a more
+	 * logical value, as it'll pin a page regardless of size. 64 is the
+	 * max value on a 4k page size architecture.
+	 */
+	reg = io_uring_setup_reg_wait(&ring, 32, &ret);
+	if (!reg) {
+		if (ret == -EINVAL) {
+			fprintf(stderr, "Kernel doesn't support registered waits\n");
+			return 1;
+		}
+		fprintf(stderr, "Registered wait: %d\n", ret);
+		return 1;
+	}
+
+	/*
+	 * Setup two distinct wait regions. Index 0 will be a 1 second wait,
+	 * and region 2 is a short wait using min_wait_usec as well. Neither
+	 * of these use a signal mask, but sigmask/sigmask_sz can be set as
+	 * well for that.
+	 */
+	reg[0].ts.tv_sec = 1;
+	reg[0].ts.tv_nsec = 0;
+	reg[0].flags = IORING_REG_WAIT_TS;
+
+	reg[1].ts.tv_sec = 0;
+	reg[1].ts.tv_nsec = 100000000LL;
+	reg[1].min_wait_usec = 10000;
+	reg[1].flags = IORING_REG_WAIT_TS;
+
+	/*
+	 * No pending completions. Wait with region 0, which should time
+	 * out after 1 second.
+	 */
+	gettimeofday(&tv, NULL);
+	ret = io_uring_submit_and_wait_reg(&ring, cqe, 1, 0);
+	if (ret == -EINVAL) {
+		fprintf(stderr, "Kernel doesn't support registered waits\n");
+		return 1;
+	} else if (ret != -ETIME) {
+		fprintf(stderr, "Wait should've timed out... %d\n", ret);
+		return 1;
+	}
+	msec = mtime_since_now(&tv);
+	if (msec < 900 || msec > 1100) {
+		fprintf(stderr, "Wait took an unexpected amount of time: %lu\n",
+			msec);
+		return 1;
+	}
+
+	/*
+	 * Now prepare two pipe reads. We'll trigger one completion quickly,
+	 * but the other one will never happen. Use min_wait_usec timeout
+	 * to abort after 10 msec of time, where the overall timeout is
+	 * otherwise 100 msec. Since we're waiting on two events, the min
+	 * timeout ends up aborting us.
+	 */
+	sqe = io_uring_get_sqe(&ring);
+	io_uring_prep_read(sqe, fds[0], b1, sizeof(b1), 0);
+	sqe = io_uring_get_sqe(&ring);
+	io_uring_prep_read(sqe, fds[0], b2, sizeof(b2), 0);
+
+	/* trigger one read */
+	ret = write(fds[1], "Hello", 5);
+	if (ret < 0) {
+		perror("write");
+		return 1;
+	}
+
+	/*
+	 * This should will wait for 2 entries, where 1 is already available.
+	 * Since we're using min_wait_usec == 10 msec here with an overall
+	 * wait of 100 msec, we expect the wait to abort after 10 msec since
+	 * one or more events are available.
+	 */
+	gettimeofday(&tv, NULL);
+	ret = io_uring_submit_and_wait_reg(&ring, cqe, 2, 1);
+	msec = mtime_since_now(&tv);
+	if (ret != 2) {
+		fprintf(stderr, "Should have submitted 2: %d\n", ret);
+		return 1;
+	}
+	if (msec < 8 || msec > 12)
+		fprintf(stderr, "min_wait_usec should take ~10 msec: %lu\n", msec);
+
+	/*
+	 * Cleanup after ourselves
+	 */
+	io_uring_queue_exit(&ring);
+	io_uring_free_reg_wait(reg, 32);
+	return 0;
+}

data/vendor/liburing/liburing.spec

@@ -1,5 +1,5 @@
 Name: liburing
-Version: 2.8
+Version: 2.9
 Release: 1%{?dist}
 Summary: Linux-native io_uring I/O access library
 License: (GPLv2 with exceptions and LGPLv2+) or MIT

data/vendor/liburing/src/include/liburing/io_uring.h

@@ -518,6 +518,7 @@ struct io_cqring_offsets {
 #define IORING_ENTER_EXT_ARG		(1U << 3)
 #define IORING_ENTER_REGISTERED_RING	(1U << 4)
 #define IORING_ENTER_ABS_TIMER		(1U << 5)
+#define IORING_ENTER_EXT_ARG_REG	(1U << 6)
 
 /*
  * Passed in for io_uring_setup(2). Copied back with updated info on success
@@ -612,6 +613,10 @@ enum io_uring_register_op {
 	/* clone registered buffers from source ring to current ring */
 	IORING_REGISTER_CLONE_BUFFERS		= 30,
 
+	IORING_REGISTER_RESIZE_RINGS		= 33,
+
+	IORING_REGISTER_CQWAIT_REG		= 34,
+
 	/* this goes last */
 	IORING_REGISTER_LAST,
 
@@ -698,13 +703,17 @@ struct io_uring_clock_register {
 };
 
 enum {
-	IORING_REGISTER_SRC_REGISTERED = 1,
+	IORING_REGISTER_SRC_REGISTERED	= (1U << 0),
+	IORING_REGISTER_DST_REPLACE	= (1U << 1),
 };
 
 struct io_uring_clone_buffers {
 	__u32	src_fd;
 	__u32	flags;
-	__u32	pad[6];
+	__u32	src_off;
+	__u32	dst_off;
+	__u32	nr;
+	__u32	pad[3];
 };
 
 struct io_uring_buf {
@@ -795,6 +804,43 @@ enum io_uring_register_restriction_op {
 	IORING_RESTRICTION_LAST
 };
 
+enum {
+	IORING_REG_WAIT_TS		= (1U << 0),
+};
+
+/*
+ * Argument for IORING_REGISTER_CQWAIT_REG, registering a region of
+ * struct io_uring_reg_wait that can be indexed when io_uring_enter(2) is
+ * called rather than pass in a wait argument structure separately.
+ */
+struct io_uring_cqwait_reg_arg {
+	__u32		flags;
+	__u32		struct_size;
+	__u32		nr_entries;
+	__u32		pad;
+	__u64		user_addr;
+	__u64		pad2[3];
+};
+
+/*
+ * Argument for io_uring_enter(2) with
+ * IORING_GETEVENTS | IORING_ENTER_EXT_ARG_REG set, where the actual argument
+ * is an index into a previously registered fixed wait region described by
+ * the below structure.
+ */
+struct io_uring_reg_wait {
+	struct __kernel_timespec	ts;
+	__u32				min_wait_usec;
+	__u32				flags;
+	__u64				sigmask;
+	__u32				sigmask_sz;
+	__u32				pad[3];
+	__u64				pad2[2];
+};
+
+/*
+ * Argument for io_uring_enter(2) with IORING_GETEVENTS | IORING_ENTER_EXT_ARG
+ */
 struct io_uring_getevents_arg {
 	__u64	sigmask;
 	__u32	sigmask_sz;

data/vendor/liburing/src/include/liburing.h

@@ -196,7 +196,16 @@ int io_uring_submit_and_wait_min_timeout(struct io_uring *ring,
 					 struct __kernel_timespec *ts,
 					 unsigned min_wait,
 					 sigset_t *sigmask);
-
+int io_uring_submit_and_wait_reg(struct io_uring *ring,
+				 struct io_uring_cqe **cqe_ptr, unsigned wait_nr,
+				 int reg_index);
+
+int io_uring_register_wait_reg(struct io_uring *ring,
+			       struct io_uring_reg_wait *reg, int nr);
+int io_uring_resize_rings(struct io_uring *ring, struct io_uring_params *p);
+int io_uring_clone_buffers_offset(struct io_uring *dst, struct io_uring *src,
+				  unsigned int dst_off, unsigned int src_off,
+				  unsigned int nr, unsigned int flags);
 int io_uring_clone_buffers(struct io_uring *dst, struct io_uring *src);
 int io_uring_register_buffers(struct io_uring *ring, const struct iovec *iovecs,
 			      unsigned nr_iovecs);
@@ -275,13 +284,20 @@ int io_uring_setup(unsigned int entries, struct io_uring_params *p);
 int io_uring_register(unsigned int fd, unsigned int opcode, const void *arg,
 		      unsigned int nr_args);
 
+/*
+ * Mapped/registered wait regions
+ */
+struct io_uring_reg_wait *io_uring_setup_reg_wait(struct io_uring *ring,
+						  unsigned nentries, int *err);
+void io_uring_free_reg_wait(struct io_uring_reg_wait *reg, unsigned nentries);
+
 /*
  * Mapped buffer ring alloc/register + unregister/free helpers
  */
 struct io_uring_buf_ring *io_uring_setup_buf_ring(struct io_uring *ring,
 						  unsigned int nentries,
 						  int bgid, unsigned int flags,
-						  int *ret);
+						  int *err);
 int io_uring_free_buf_ring(struct io_uring *ring, struct io_uring_buf_ring *br,
 			   unsigned int nentries, int bgid);
 
@@ -1292,6 +1308,16 @@ IOURINGINLINE void io_uring_prep_ftruncate(struct io_uring_sqe *sqe,
 }
 #endif
 
+IOURINGINLINE void io_uring_prep_cmd_discard(struct io_uring_sqe *sqe,
+					     int fd,
+					     uint64_t offset, uint64_t nbytes)
+{
+	io_uring_prep_rw(IORING_OP_URING_CMD, sqe, fd, 0, 0, 0);
+	sqe->cmd_op = BLOCK_URING_CMD_DISCARD;
+	sqe->addr = offset;
+	sqe->addr3 = nbytes;
+}
+
 /*
  * Returns number of unconsumed (if SQPOLL) or unsubmitted entries exist in
  * the SQ ring

data/vendor/liburing/src/int_flags.h

@@ -2,10 +2,17 @@
 #ifndef LIBURING_INT_FLAGS
 #define LIBURING_INT_FLAGS
 
+#define INT_FLAGS_MASK		(IORING_ENTER_REGISTERED_RING)
+
 enum {
-	INT_FLAG_REG_RING	= 1,
-	INT_FLAG_REG_REG_RING	= 2,
-	INT_FLAG_APP_MEM	= 4,
+	INT_FLAG_REG_RING	= IORING_ENTER_REGISTERED_RING,
+	INT_FLAG_REG_REG_RING	= 1,
+	INT_FLAG_APP_MEM	= 2,
 };
 
+static inline int ring_enter_flags(struct io_uring *ring)
+{
+	return ring->int_flags & INT_FLAGS_MASK;
+}
+
 #endif

data/vendor/liburing/src/liburing-ffi.map

@@ -211,6 +211,17 @@ LIBURING_2.8 {
 		io_uring_submit_and_wait_min_timeout;
 		io_uring_wait_cqes_min_timeout;
 		io_uring_clone_buffers;
-        io_uring_prep_open;
-        io_uring_prep_open_direct;
+		io_uring_prep_cmd_discard;
+		io_uring_prep_open;
+		io_uring_prep_open_direct;
 } LIBURING_2.7;
+
+LIBURING_2.9 {
+	global:
+		io_uring_resize_rings;
+		io_uring_register_wait_reg;
+		io_uring_submit_and_wait_reg;
+		io_uring_free_reg_wait;
+		io_uring_setup_reg_wait;
+		io_uring_clone_buffers_offset;
+} LIBURING_2.8;

data/vendor/liburing/src/liburing.map

@@ -103,3 +103,12 @@ LIBURING_2.8 {
 		io_uring_wait_cqes_min_timeout;
 		io_uring_clone_buffers;
 } LIBURING_2.7;
+
+LIBURING_2.9 {
+		io_uring_resize_rings;
+		io_uring_register_wait_reg;
+		io_uring_submit_and_wait_reg;
+		io_uring_free_reg_wait;
+		io_uring_setup_reg_wait;
+		io_uring_clone_buffers_offset;
+} LIBURING_2.8;

data/vendor/liburing/src/queue.c

@@ -6,7 +6,6 @@
 #include "liburing.h"
 #include "int_flags.h"
 #include "liburing/sanitize.h"
-#include "liburing/compat.h"
 #include "liburing/io_uring.h"
 
 /*
@@ -70,7 +69,7 @@ static int _io_uring_get_cqe(struct io_uring *ring,
 
 	do {
 		bool need_enter = false;
-		unsigned flags = 0;
+		unsigned flags = ring_enter_flags(ring);
 		unsigned nr_available;
 		int ret;
 
@@ -94,7 +93,7 @@ static int _io_uring_get_cqe(struct io_uring *ring,
 			need_enter = true;
 		}
 		if (data->wait_nr > nr_available || need_enter) {
-			flags = IORING_ENTER_GETEVENTS | data->get_flags;
+			flags |= IORING_ENTER_GETEVENTS | data->get_flags;
 			need_enter = true;
 		}
 		if (sq_ring_needs_enter(ring, data->submit, &flags))
@@ -109,8 +108,6 @@ static int _io_uring_get_cqe(struct io_uring *ring,
 			break;
 		}
 
-		if (ring->int_flags & INT_FLAG_REG_RING)
-			flags |= IORING_ENTER_REGISTERED_RING;
 		ret = __sys_io_uring_enter2(ring->enter_ring_fd, data->submit,
 					    data->wait_nr, flags, data->arg,
 					    data->sz);
@@ -149,10 +146,8 @@ int __io_uring_get_cqe(struct io_uring *ring, struct io_uring_cqe **cqe_ptr,
 
 int io_uring_get_events(struct io_uring *ring)
 {
-	int flags = IORING_ENTER_GETEVENTS;
+	int flags = IORING_ENTER_GETEVENTS | ring_enter_flags(ring);
 
-	if (ring->int_flags & INT_FLAG_REG_RING)
-		flags |= IORING_ENTER_REGISTERED_RING;
 	return __sys_io_uring_enter(ring->enter_ring_fd, 0, 0, flags, NULL);
 }
 
@@ -325,6 +320,26 @@ int io_uring_wait_cqes_min_timeout(struct io_uring *ring,
 					sigmask);
 }
 
+int io_uring_submit_and_wait_reg(struct io_uring *ring,
+				 struct io_uring_cqe **cqe_ptr,
+				 unsigned wait_nr, int reg_index)
+{
+	struct get_data data = {
+		.submit		= __io_uring_flush_sq(ring),
+		.wait_nr	= wait_nr,
+		.get_flags	= IORING_ENTER_EXT_ARG |
+				  IORING_ENTER_EXT_ARG_REG,
+		.sz		= sizeof(struct io_uring_reg_wait),
+		.has_ts		= true,
+		.arg		= (void *) (uintptr_t) reg_index,
+	};
+
+	if (!(ring->features & IORING_FEAT_EXT_ARG))
+		return -EINVAL;
+
+	return _io_uring_get_cqe(ring, cqe_ptr, &data);
+}
+
 static int __io_uring_submit_and_wait_timeout(struct io_uring *ring,
 			struct io_uring_cqe **cqe_ptr, unsigned wait_nr,
 			struct __kernel_timespec *ts,
@@ -403,17 +418,14 @@ static int __io_uring_submit(struct io_uring *ring, unsigned submitted,
 			     unsigned wait_nr, bool getevents)
 {
 	bool cq_needs_enter = getevents || wait_nr || cq_ring_needs_enter(ring);
-	unsigned flags;
+	unsigned flags = ring_enter_flags(ring);
 	int ret;
 
 	liburing_sanitize_ring(ring);
 
-	flags = 0;
 	if (sq_ring_needs_enter(ring, submitted, &flags) || cq_needs_enter) {
 		if (cq_needs_enter)
 			flags |= IORING_ENTER_GETEVENTS;
-		if (ring->int_flags & INT_FLAG_REG_RING)
-			flags |= IORING_ENTER_REGISTERED_RING;
 
 		ret = __sys_io_uring_enter(ring->enter_ring_fd, submitted,
 					   wait_nr, flags, NULL);
@@ -462,10 +474,7 @@ struct io_uring_sqe *io_uring_get_sqe(struct io_uring *ring)
 
 int __io_uring_sqring_wait(struct io_uring *ring)
 {
-	int flags = IORING_ENTER_SQ_WAIT;
-
-	if (ring->int_flags & INT_FLAG_REG_RING)
-		flags |= IORING_ENTER_REGISTERED_RING;
+	int flags = IORING_ENTER_SQ_WAIT | ring_enter_flags(ring);
 
 	return __sys_io_uring_enter(ring->enter_ring_fd, 0, 0, flags, NULL);
 }

data/vendor/liburing/src/register.c

@@ -4,8 +4,8 @@
 #include "lib.h"
 #include "syscall.h"
 #include "liburing.h"
+#include "setup.h"
 #include "int_flags.h"
-#include "liburing/compat.h"
 #include "liburing/io_uring.h"
 #include "liburing/sanitize.h"
 
@@ -395,16 +395,85 @@ int io_uring_register_clock(struct io_uring *ring,
 	return do_register(ring, IORING_REGISTER_CLOCK, arg, 0);
 }
 
-int io_uring_clone_buffers(struct io_uring *dst, struct io_uring *src)
+int io_uring_clone_buffers_offset(struct io_uring *dst, struct io_uring *src,
+				  unsigned int dst_off, unsigned int src_off,
+				  unsigned int nr, unsigned int flags)
 {
-	struct io_uring_clone_buffers buf = { .src_fd = src->ring_fd, };
+	struct io_uring_clone_buffers buf = {
+		.src_fd		= src->ring_fd,
+		.flags		= flags,
+		.src_off	= src_off,
+		.dst_off	= dst_off,
+		.nr		= nr,
+	};
 
 	if (src->int_flags & INT_FLAG_REG_REG_RING) {
 		buf.src_fd = src->enter_ring_fd;
-		buf.flags = IORING_REGISTER_SRC_REGISTERED;
+		buf.flags |= IORING_REGISTER_SRC_REGISTERED;
 	} else {
 		buf.src_fd = src->ring_fd;
 	}
 
 	return do_register(dst, IORING_REGISTER_CLONE_BUFFERS, &buf, 1);
 }
+
+int io_uring_clone_buffers(struct io_uring *dst, struct io_uring *src)
+{
+	return io_uring_clone_buffers_offset(dst, src, 0, 0, 0, 0);
+}
+
+int io_uring_resize_rings(struct io_uring *ring, struct io_uring_params *p)
+{
+	unsigned sq_head, sq_tail;
+	int ret;
+
+	if (ring->flags & IORING_SETUP_NO_MMAP)
+		return -EINVAL;
+
+	memset(&p->sq_off, 0, sizeof(p->sq_off));
+	memset(&p->cq_off, 0, sizeof(p->cq_off));
+
+	ret = do_register(ring, IORING_REGISTER_RESIZE_RINGS, p, 1);
+	if (ret < 0)
+		goto out;
+
+	sq_head = ring->sq.sqe_head;
+	sq_tail = ring->sq.sqe_tail;
+	io_uring_unmap_rings(&ring->sq, &ring->cq);
+	memset(&ring->sq, 0, sizeof(ring->sq));
+	memset(&ring->cq, 0, sizeof(ring->cq));
+	ret = io_uring_mmap(ring->ring_fd, p, &ring->sq, &ring->cq);
+	if (ret)
+		goto out;
+
+	ring->sq.sqe_head = sq_head;
+	ring->sq.sqe_tail = sq_tail;
+
+	/*
+	 * Directly map SQ slots to SQEs
+	 */
+	if (!(p->flags & IORING_SETUP_NO_SQARRAY)) {
+		unsigned index;
+
+		for (index = 0; index < p->sq_entries; index++)
+			ring->sq.array[index] = index;
+	}
+
+	/* clear for next use */
+out:
+	p->flags = 0;
+	return ret;
+}
+
+int io_uring_register_wait_reg(struct io_uring *ring,
+			       struct io_uring_reg_wait *reg, int nr)
+{
+	struct io_uring_cqwait_reg_arg arg = {
+		.flags		= 0,
+		.struct_size	= sizeof(*reg),
+		.nr_entries	= nr,
+		.user_addr	= (unsigned long) (uintptr_t) reg,
+	};
+
+	return do_register(ring, IORING_REGISTER_CQWAIT_REG, &arg, 1);
+}