uringmachine 0.3 → 0.5

data/vendor/liburing/liburing.pc.in

@@ -9,4 +9,4 @@ Description: io_uring library
 URL: https://git.kernel.dk/cgit/liburing/
 
 Libs: -L${libdir} -luring
-Cflags: -I${includedir} -D_GNU_SOURCE
+Cflags: -I${includedir}

data/vendor/liburing/liburing.spec

@@ -1,5 +1,5 @@
 Name: liburing
-Version: 2.8
+Version: 2.9
 Release: 1%{?dist}
 Summary: Linux-native io_uring I/O access library
 License: (GPLv2 with exceptions and LGPLv2+) or MIT

data/vendor/liburing/src/Makefile

@@ -16,8 +16,6 @@ override CFLAGS += -Wno-unused-parameter \
 	$(LIBURING_CFLAGS)
 SO_CFLAGS=-fPIC $(CFLAGS)
 L_CFLAGS=$(CFLAGS)
-LINK_FLAGS=-Wl,-z,defs
-LINK_FLAGS+=$(LDFLAGS)
 ENABLE_SHARED ?= 1
 
 soname=liburing.so.$(VERSION_MAJOR)
@@ -39,6 +37,13 @@ ifneq ($(MAKECMDGOALS),clean)
 include ../config-host.mak
 endif
 
+ifeq ($(CONFIG_USE_SANITIZER),y)
+	LINK_FLAGS=
+else
+	LINK_FLAGS=-Wl,-z,defs
+endif
+LINK_FLAGS+=$(LDFLAGS)
+
 all: $(all_targets)
 
 liburing_srcs := setup.c queue.c register.c syscall.c version.c
@@ -50,6 +55,13 @@ ifeq ($(CONFIG_NOLIBC),y)
 	override LINK_FLAGS += -nostdlib -nodefaultlibs $(libgcc_link_flag)
 endif
 
+ifeq ($(CONFIG_USE_SANITIZER),y)
+	override CFLAGS += -fsanitize=address,undefined -g -fno-omit-frame-pointer -fno-optimize-sibling-calls
+	override CPPFLAGS += -fsanitize=address,undefined -g -fno-omit-frame-pointer -fno-optimize-sibling-calls
+	override LINK_FLAGS += -fsanitize=address,undefined
+	liburing_srcs += sanitize.c
+endif
+
 override CPPFLAGS += -MT "$@" -MMD -MP -MF "$@.d"
 liburing_objs := $(patsubst %.c,%.ol,$(liburing_srcs))
 liburing_sobjs := $(patsubst %.c,%.os,$(liburing_srcs))
@@ -89,6 +101,7 @@ install: $(all_targets)
 	install -D -m 644 include/liburing.h $(includedir)/liburing.h
 	install -D -m 644 include/liburing/compat.h $(includedir)/liburing/compat.h
 	install -D -m 644 include/liburing/barrier.h $(includedir)/liburing/barrier.h
+	install -D -m 644 include/liburing/sanitize.h $(includedir)/liburing/sanitize.h
 	install -D -m 644 include/liburing/io_uring_version.h $(includedir)/liburing/io_uring_version.h
 	install -D -m 644 liburing.a $(libdevdir)/liburing.a
 	install -D -m 644 liburing-ffi.a $(libdevdir)/liburing-ffi.a
@@ -106,6 +119,7 @@ uninstall:
 	@rm -f $(includedir)/liburing.h
 	@rm -f $(includedir)/liburing/compat.h
 	@rm -f $(includedir)/liburing/barrier.h
+	@rm -f $(includedir)/liburing/sanitize.h
 	@rm -f $(includedir)/liburing/io_uring_version.h
 	@rm -f $(libdevdir)/liburing.a
 	@rm -f $(libdevdir)/liburing-ffi.a

data/vendor/liburing/src/include/liburing/io_uring.h

@@ -440,11 +440,21 @@ struct io_uring_cqe {
  * IORING_CQE_F_SOCK_NONEMPTY	If set, more data to read after socket recv
  * IORING_CQE_F_NOTIF	Set for notification CQEs. Can be used to distinct
  * 			them from sends.
+ * IORING_CQE_F_BUF_MORE If set, the buffer ID set in the completion will get
+ *			more completions. In other words, the buffer is being
+ *			partially consumed, and will be used by the kernel for
+ *			more completions. This is only set for buffers used via
+ *			the incremental buffer consumption, as provided by
+ *			a ring buffer setup with IOU_PBUF_RING_INC. For any
+ *			other provided buffer type, all completions with a
+ *			buffer passed back is automatically returned to the
+ *			application.
  */
 #define IORING_CQE_F_BUFFER		(1U << 0)
 #define IORING_CQE_F_MORE		(1U << 1)
 #define IORING_CQE_F_SOCK_NONEMPTY	(1U << 2)
 #define IORING_CQE_F_NOTIF		(1U << 3)
+#define IORING_CQE_F_BUF_MORE		(1U << 4)
 
 #define IORING_CQE_BUFFER_SHIFT		16
 
@@ -508,6 +518,7 @@ struct io_cqring_offsets {
 #define IORING_ENTER_EXT_ARG		(1U << 3)
 #define IORING_ENTER_REGISTERED_RING	(1U << 4)
 #define IORING_ENTER_ABS_TIMER		(1U << 5)
+#define IORING_ENTER_EXT_ARG_REG	(1U << 6)
 
 /*
  * Passed in for io_uring_setup(2). Copied back with updated info on success
@@ -599,6 +610,13 @@ enum io_uring_register_op {
 
 	IORING_REGISTER_CLOCK			= 29,
 
+	/* clone registered buffers from source ring to current ring */
+	IORING_REGISTER_CLONE_BUFFERS		= 30,
+
+	IORING_REGISTER_RESIZE_RINGS		= 33,
+
+	IORING_REGISTER_CQWAIT_REG		= 34,
+
 	/* this goes last */
 	IORING_REGISTER_LAST,
 
@@ -684,6 +702,20 @@ struct io_uring_clock_register {
 	__u32	__resv[3];
 };
 
+enum {
+	IORING_REGISTER_SRC_REGISTERED	= (1U << 0),
+	IORING_REGISTER_DST_REPLACE	= (1U << 1),
+};
+
+struct io_uring_clone_buffers {
+	__u32	src_fd;
+	__u32	flags;
+	__u32	src_off;
+	__u32	dst_off;
+	__u32	nr;
+	__u32	pad[3];
+};
+
 struct io_uring_buf {
 	__u64	addr;
 	__u32	len;
@@ -716,9 +748,17 @@ struct io_uring_buf_ring {
  *			mmap(2) with the offset set as:
  *			IORING_OFF_PBUF_RING | (bgid << IORING_OFF_PBUF_SHIFT)
  *			to get a virtual mapping for the ring.
+ * IOU_PBUF_RING_INC:	If set, buffers consumed from this buffer ring can be
+ *			consumed incrementally. Normally one (or more) buffers
+ *			are fully consumed. With incremental consumptions, it's
+ *			feasible to register big ranges of buffers, and each
+ *			use of it will consume only as much as it needs. This
+ *			requires that both the kernel and application keep
+ *			track of where the current read/recv index is at.
  */
 enum io_uring_register_pbuf_ring_flags {
 	IOU_PBUF_RING_MMAP	= 1,
+	IOU_PBUF_RING_INC	= 2,
 };
 
 /* argument for IORING_(UN)REGISTER_PBUF_RING */
@@ -764,6 +804,43 @@ enum io_uring_register_restriction_op {
 	IORING_RESTRICTION_LAST
 };
 
+enum {
+	IORING_REG_WAIT_TS		= (1U << 0),
+};
+
+/*
+ * Argument for IORING_REGISTER_CQWAIT_REG, registering a region of
+ * struct io_uring_reg_wait that can be indexed when io_uring_enter(2) is
+ * called rather than pass in a wait argument structure separately.
+ */
+struct io_uring_cqwait_reg_arg {
+	__u32		flags;
+	__u32		struct_size;
+	__u32		nr_entries;
+	__u32		pad;
+	__u64		user_addr;
+	__u64		pad2[3];
+};
+
+/*
+ * Argument for io_uring_enter(2) with
+ * IORING_GETEVENTS | IORING_ENTER_EXT_ARG_REG set, where the actual argument
+ * is an index into a previously registered fixed wait region described by
+ * the below structure.
+ */
+struct io_uring_reg_wait {
+	struct __kernel_timespec	ts;
+	__u32				min_wait_usec;
+	__u32				flags;
+	__u64				sigmask;
+	__u32				sigmask_sz;
+	__u32				pad[3];
+	__u64				pad2[2];
+};
+
+/*
+ * Argument for io_uring_enter(2) with IORING_GETEVENTS | IORING_ENTER_EXT_ARG
+ */
 struct io_uring_getevents_arg {
 	__u64	sigmask;
 	__u32	sigmask_sz;

data/vendor/liburing/src/include/liburing/sanitize.h

@@ -0,0 +1,39 @@
+/* SPDX-License-Identifier: MIT */
+#ifndef LIBURING_SANITIZE_H
+#define LIBURING_SANITIZE_H
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+struct io_uring;
+struct iovec;
+
+#if defined(CONFIG_USE_SANITIZER)
+void liburing_sanitize_ring(struct io_uring *ring);
+void liburing_sanitize_address(const void *addr);
+void liburing_sanitize_region(const void *addr, unsigned int len);
+void liburing_sanitize_iovecs(const struct iovec *iovecs, unsigned nr);
+#else
+#define __maybe_unused	__attribute__((__unused__))
+static inline void liburing_sanitize_ring(struct io_uring __maybe_unused *ring)
+{
+}
+static inline void liburing_sanitize_address(const void __maybe_unused *addr)
+{
+}
+static inline void liburing_sanitize_region(const void __maybe_unused *addr,
+					    unsigned int __maybe_unused len)
+{
+}
+static inline void liburing_sanitize_iovecs(const struct iovec __maybe_unused *iovecs,
+					    unsigned __maybe_unused nr)
+{
+}
+#endif
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif

data/vendor/liburing/src/include/liburing.h

@@ -19,6 +19,7 @@
 #include "liburing/io_uring_version.h"
 #include "liburing/barrier.h"
 
+
 #ifndef uring_unlikely
 #define uring_unlikely(cond)	__builtin_expect(!!(cond), 0)
 #endif
@@ -195,7 +196,17 @@ int io_uring_submit_and_wait_min_timeout(struct io_uring *ring,
 					 struct __kernel_timespec *ts,
 					 unsigned min_wait,
 					 sigset_t *sigmask);
-
+int io_uring_submit_and_wait_reg(struct io_uring *ring,
+				 struct io_uring_cqe **cqe_ptr, unsigned wait_nr,
+				 int reg_index);
+
+int io_uring_register_wait_reg(struct io_uring *ring,
+			       struct io_uring_reg_wait *reg, int nr);
+int io_uring_resize_rings(struct io_uring *ring, struct io_uring_params *p);
+int io_uring_clone_buffers_offset(struct io_uring *dst, struct io_uring *src,
+				  unsigned int dst_off, unsigned int src_off,
+				  unsigned int nr, unsigned int flags);
+int io_uring_clone_buffers(struct io_uring *dst, struct io_uring *src);
 int io_uring_register_buffers(struct io_uring *ring, const struct iovec *iovecs,
 			      unsigned nr_iovecs);
 int io_uring_register_buffers_tags(struct io_uring *ring,
@@ -232,8 +243,10 @@ int io_uring_register_restrictions(struct io_uring *ring,
 				   unsigned int nr_res);
 int io_uring_enable_rings(struct io_uring *ring);
 int __io_uring_sqring_wait(struct io_uring *ring);
+#ifdef _GNU_SOURCE
 int io_uring_register_iowq_aff(struct io_uring *ring, size_t cpusz,
 				const cpu_set_t *mask);
+#endif
 int io_uring_unregister_iowq_aff(struct io_uring *ring);
 int io_uring_register_iowq_max_workers(struct io_uring *ring,
 				       unsigned int *values);
@@ -271,13 +284,20 @@ int io_uring_setup(unsigned int entries, struct io_uring_params *p);
 int io_uring_register(unsigned int fd, unsigned int opcode, const void *arg,
 		      unsigned int nr_args);
 
+/*
+ * Mapped/registered wait regions
+ */
+struct io_uring_reg_wait *io_uring_setup_reg_wait(struct io_uring *ring,
+						  unsigned nentries, int *err);
+void io_uring_free_reg_wait(struct io_uring_reg_wait *reg, unsigned nentries);
+
 /*
  * Mapped buffer ring alloc/register + unregister/free helpers
  */
 struct io_uring_buf_ring *io_uring_setup_buf_ring(struct io_uring *ring,
 						  unsigned int nentries,
 						  int bgid, unsigned int flags,
-						  int *ret);
+						  int *err);
 int io_uring_free_buf_ring(struct io_uring *ring, struct io_uring_buf_ring *br,
 			   unsigned int nentries, int bgid);
 
@@ -301,15 +321,22 @@ int __io_uring_get_cqe(struct io_uring *ring,
 #define io_uring_cqe_index(ring,ptr,mask)				\
 	(((ptr) & (mask)) << io_uring_cqe_shift(ring))
 
+/*
+ * NOTE: we should just get rid of the 'head' being passed in here, it doesn't
+ * serve a purpose anymore. The below is a bit of a work-around to ensure that
+ * the compiler doesn't complain about 'head' being unused (or only written,
+ * never read), as we use a local iterator for both the head and tail tracking.
+ */
 #define io_uring_for_each_cqe(ring, head, cqe)				\
 	/*								\
 	 * io_uring_smp_load_acquire() enforces the order of tail	\
 	 * and CQE reads.						\
 	 */								\
-	for (head = *(ring)->cq.khead;					\
-	     (cqe = (head != io_uring_smp_load_acquire((ring)->cq.ktail) ? \
-		&(ring)->cq.cqes[io_uring_cqe_index(ring, head, (ring)->cq.ring_mask)] : NULL)); \
-	     head++)							\
+	for (__u32 __HEAD__ = (head) = *(ring)->cq.khead,		\
+	     __TAIL__ = io_uring_smp_load_acquire((ring)->cq.ktail);	\
+	     (cqe = ((head) != __TAIL__ ?				\
+	     &(ring)->cq.cqes[io_uring_cqe_index(ring, __HEAD__, (ring)->cq.ring_mask)] : NULL)); \
+	     (head) = ++__HEAD__)
 
 /*
  * Must be called after io_uring_for_each_cqe()
@@ -734,6 +761,20 @@ IOURINGINLINE void io_uring_prep_openat_direct(struct io_uring_sqe *sqe,
 	__io_uring_set_target_fixed_file(sqe, file_index);
 }
 
+IOURINGINLINE void io_uring_prep_open(struct io_uring_sqe *sqe,
+					const char *path, int flags, mode_t mode)
+{
+	io_uring_prep_openat(sqe, AT_FDCWD, path, flags, mode);
+}
+
+/* open directly into the fixed file table */
+IOURINGINLINE void io_uring_prep_open_direct(struct io_uring_sqe *sqe,
+							const char *path, int flags, mode_t mode,
+							unsigned file_index)
+{
+	io_uring_prep_openat_direct(sqe, AT_FDCWD, path, flags, mode, file_index);
+}
+
 IOURINGINLINE void io_uring_prep_close(struct io_uring_sqe *sqe, int fd)
 {
 	io_uring_prep_rw(IORING_OP_CLOSE, sqe, fd, NULL, 0, 0);
@@ -1259,11 +1300,23 @@ IOURINGINLINE void io_uring_prep_fixed_fd_install(struct io_uring_sqe *sqe,
 	sqe->install_fd_flags = flags;
 }
 
+#ifdef _GNU_SOURCE
 IOURINGINLINE void io_uring_prep_ftruncate(struct io_uring_sqe *sqe,
 				       int fd, loff_t len)
 {
 	io_uring_prep_rw(IORING_OP_FTRUNCATE, sqe, fd, 0, 0, len);
 }
+#endif
+
+IOURINGINLINE void io_uring_prep_cmd_discard(struct io_uring_sqe *sqe,
+					     int fd,
+					     uint64_t offset, uint64_t nbytes)
+{
+	io_uring_prep_rw(IORING_OP_URING_CMD, sqe, fd, 0, 0, 0);
+	sqe->cmd_op = BLOCK_URING_CMD_DISCARD;
+	sqe->addr = offset;
+	sqe->addr3 = nbytes;
+}
 
 /*
  * Returns number of unconsumed (if SQPOLL) or unsubmitted entries exist in

data/vendor/liburing/src/int_flags.h

@@ -2,10 +2,17 @@
 #ifndef LIBURING_INT_FLAGS
 #define LIBURING_INT_FLAGS
 
+#define INT_FLAGS_MASK		(IORING_ENTER_REGISTERED_RING)
+
 enum {
-	INT_FLAG_REG_RING	= 1,
-	INT_FLAG_REG_REG_RING	= 2,
-	INT_FLAG_APP_MEM	= 4,
+	INT_FLAG_REG_RING	= IORING_ENTER_REGISTERED_RING,
+	INT_FLAG_REG_REG_RING	= 1,
+	INT_FLAG_APP_MEM	= 2,
 };
 
+static inline int ring_enter_flags(struct io_uring *ring)
+{
+	return ring->int_flags & INT_FLAGS_MASK;
+}
+
 #endif

data/vendor/liburing/src/liburing-ffi.map

@@ -208,4 +208,20 @@ LIBURING_2.7 {
 LIBURING_2.8 {
 	global:
 		io_uring_register_clock;
+		io_uring_submit_and_wait_min_timeout;
+		io_uring_wait_cqes_min_timeout;
+		io_uring_clone_buffers;
+		io_uring_prep_cmd_discard;
+		io_uring_prep_open;
+		io_uring_prep_open_direct;
 } LIBURING_2.7;
+
+LIBURING_2.9 {
+	global:
+		io_uring_resize_rings;
+		io_uring_register_wait_reg;
+		io_uring_submit_and_wait_reg;
+		io_uring_free_reg_wait;
+		io_uring_setup_reg_wait;
+		io_uring_clone_buffers_offset;
+} LIBURING_2.8;

data/vendor/liburing/src/liburing.map

@@ -101,4 +101,14 @@ LIBURING_2.8 {
 		io_uring_register_clock;
 		io_uring_submit_and_wait_min_timeout;
 		io_uring_wait_cqes_min_timeout;
+		io_uring_clone_buffers;
 } LIBURING_2.7;
+
+LIBURING_2.9 {
+		io_uring_resize_rings;
+		io_uring_register_wait_reg;
+		io_uring_submit_and_wait_reg;
+		io_uring_free_reg_wait;
+		io_uring_setup_reg_wait;
+		io_uring_clone_buffers_offset;
+} LIBURING_2.8;

data/vendor/liburing/src/queue.c

@@ -5,7 +5,7 @@
 #include "syscall.h"
 #include "liburing.h"
 #include "int_flags.h"
-#include "liburing/compat.h"
+#include "liburing/sanitize.h"
 #include "liburing/io_uring.h"
 
 /*
@@ -69,7 +69,7 @@ static int _io_uring_get_cqe(struct io_uring *ring,
 
 	do {
 		bool need_enter = false;
-		unsigned flags = 0;
+		unsigned flags = ring_enter_flags(ring);
 		unsigned nr_available;
 		int ret;
 
@@ -93,7 +93,7 @@ static int _io_uring_get_cqe(struct io_uring *ring,
 			need_enter = true;
 		}
 		if (data->wait_nr > nr_available || need_enter) {
-			flags = IORING_ENTER_GETEVENTS | data->get_flags;
+			flags |= IORING_ENTER_GETEVENTS | data->get_flags;
 			need_enter = true;
 		}
 		if (sq_ring_needs_enter(ring, data->submit, &flags))
@@ -108,8 +108,6 @@ static int _io_uring_get_cqe(struct io_uring *ring,
 			break;
 		}
 
-		if (ring->int_flags & INT_FLAG_REG_RING)
-			flags |= IORING_ENTER_REGISTERED_RING;
 		ret = __sys_io_uring_enter2(ring->enter_ring_fd, data->submit,
 					    data->wait_nr, flags, data->arg,
 					    data->sz);
@@ -148,10 +146,8 @@ int __io_uring_get_cqe(struct io_uring *ring, struct io_uring_cqe **cqe_ptr,
 
 int io_uring_get_events(struct io_uring *ring)
 {
-	int flags = IORING_ENTER_GETEVENTS;
+	int flags = IORING_ENTER_GETEVENTS | ring_enter_flags(ring);
 
-	if (ring->int_flags & INT_FLAG_REG_RING)
-		flags |= IORING_ENTER_REGISTERED_RING;
 	return __sys_io_uring_enter(ring->enter_ring_fd, 0, 0, flags, NULL);
 }
 
@@ -324,6 +320,26 @@ int io_uring_wait_cqes_min_timeout(struct io_uring *ring,
 					sigmask);
 }
 
+int io_uring_submit_and_wait_reg(struct io_uring *ring,
+				 struct io_uring_cqe **cqe_ptr,
+				 unsigned wait_nr, int reg_index)
+{
+	struct get_data data = {
+		.submit		= __io_uring_flush_sq(ring),
+		.wait_nr	= wait_nr,
+		.get_flags	= IORING_ENTER_EXT_ARG |
+				  IORING_ENTER_EXT_ARG_REG,
+		.sz		= sizeof(struct io_uring_reg_wait),
+		.has_ts		= true,
+		.arg		= (void *) (uintptr_t) reg_index,
+	};
+
+	if (!(ring->features & IORING_FEAT_EXT_ARG))
+		return -EINVAL;
+
+	return _io_uring_get_cqe(ring, cqe_ptr, &data);
+}
+
 static int __io_uring_submit_and_wait_timeout(struct io_uring *ring,
 			struct io_uring_cqe **cqe_ptr, unsigned wait_nr,
 			struct __kernel_timespec *ts,
@@ -402,15 +418,14 @@ static int __io_uring_submit(struct io_uring *ring, unsigned submitted,
 			     unsigned wait_nr, bool getevents)
 {
 	bool cq_needs_enter = getevents || wait_nr || cq_ring_needs_enter(ring);
-	unsigned flags;
+	unsigned flags = ring_enter_flags(ring);
 	int ret;
 
-	flags = 0;
+	liburing_sanitize_ring(ring);
+
 	if (sq_ring_needs_enter(ring, submitted, &flags) || cq_needs_enter) {
 		if (cq_needs_enter)
 			flags |= IORING_ENTER_GETEVENTS;
-		if (ring->int_flags & INT_FLAG_REG_RING)
-			flags |= IORING_ENTER_REGISTERED_RING;
 
 		ret = __sys_io_uring_enter(ring->enter_ring_fd, submitted,
 					   wait_nr, flags, NULL);
@@ -459,10 +474,7 @@ struct io_uring_sqe *io_uring_get_sqe(struct io_uring *ring)
 
 int __io_uring_sqring_wait(struct io_uring *ring)
 {
-	int flags = IORING_ENTER_SQ_WAIT;
-
-	if (ring->int_flags & INT_FLAG_REG_RING)
-		flags |= IORING_ENTER_REGISTERED_RING;
+	int flags = IORING_ENTER_SQ_WAIT | ring_enter_flags(ring);
 
 	return __sys_io_uring_enter(ring->enter_ring_fd, 0, 0, flags, NULL);
 }