uringmachine 0.3 → 0.4

data/vendor/liburing/examples/kdigest.c

@@ -0,0 +1,405 @@
+/* SPDX-License-Identifier: MIT */
+/*
+ * Proof-of-concept for doing file digests using the kernel's AF_ALG API.
+ * Needs a bit of error handling.
+ */
+#include <stdio.h>
+#include <fcntl.h>
+#include <string.h>
+#include <stdlib.h>
+#include <unistd.h>
+#include <assert.h>
+#include <errno.h>
+#include <inttypes.h>
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <sys/ioctl.h>
+#include <linux/if_alg.h>
+#include "liburing.h"
+
+#define QD		64
+#define WAIT_BATCH	(QD / 8)
+#define BS		(64*1024)
+
+#define BGID		1
+#define BID_MASK	(QD - 1)
+
+enum req_state {
+	IO_INIT = 0,
+	IO_READ,
+	IO_READ_COMPLETE,
+	IO_WRITE,
+	IO_WRITE_COMPLETE,
+};
+
+struct req {
+	off_t offset;
+	enum req_state state;
+	struct iovec iov;
+};
+
+struct kdigest {
+	struct io_uring ring;
+	struct io_uring_buf_ring *br;
+	struct req reqs[QD];
+	/* heap allocated, aligned QD*BS buffer */
+	uint8_t *bufs;
+};
+
+static int infd, outfd;
+
+static int get_file_size(int fd, size_t *size)
+{
+	struct stat st;
+
+	if (fstat(fd, &st) < 0)
+		return -1;
+	if (S_ISREG(st.st_mode)) {
+		*size = st.st_size;
+	} else if (S_ISBLK(st.st_mode)) {
+		unsigned long long bytes;
+
+		if (ioctl(fd, BLKGETSIZE64, &bytes) != 0)
+			return -1;
+
+		*size = bytes;
+	} else {
+		return -1;
+	}
+
+	return 0;
+}
+
+static int reap_completions(struct io_uring *ring, int *inflight,
+			    size_t *outsize)
+{
+	struct io_uring_cqe *cqe;
+	unsigned head;
+	int ret = 0, nr;
+
+	nr = 0;
+	io_uring_for_each_cqe(ring, head, cqe) {
+		struct req *req;
+
+		req = io_uring_cqe_get_data(cqe);
+		assert(req->state == IO_READ || req->state == IO_WRITE);
+		if (cqe->res < 0) {
+			fprintf(stderr, "%s: cqe error %d\n",
+				req->state == IO_WRITE ? "send" : "read",
+				cqe->res);
+			*outsize = 0;
+			ret = 1;
+			break;
+		}
+
+		(*inflight)--;
+		req->state++;
+		if (req->state == IO_WRITE_COMPLETE)
+			*outsize -= cqe->res;
+		nr++;
+	}
+
+	io_uring_cq_advance(ring, nr);
+	return ret;
+}
+
+/*
+ * Add buffers to the outgoing ring, and submit a single bundle send that
+ * will finish when all of them have completed.
+ */
+static void submit_sends_br(struct kdigest *kdigest, int *write_idx,
+			    int *inflight)
+{
+	struct io_uring_buf_ring *br = kdigest->br;
+	struct req *req, *first_req = NULL;
+	struct io_uring_sqe *sqe;
+	int nr = 0;
+
+	/*
+	 * Find any completed reads, and add the buffers to the outgoing
+	 * send ring. That will serialize the data sent.
+	 */
+	while (kdigest->reqs[*write_idx].state == IO_READ_COMPLETE) {
+		req = &kdigest->reqs[*write_idx];
+		io_uring_buf_ring_add(br, req->iov.iov_base, req->iov.iov_len,
+					*write_idx, BID_MASK, nr++);
+		/*
+		 * Mark as a write/send if it's the first one, that serve
+		 * as the "barrier" in the array. The rest can be marked
+		 * complete upfront, if there's more in this bundle, as
+		 * the first will serve a the stopping point.
+		 */
+		if (!first_req) {
+			req->state = IO_WRITE;
+			first_req = req;
+		} else {
+			req->state = IO_WRITE_COMPLETE;
+		}
+		*write_idx = (*write_idx + 1) % QD;
+	}
+
+	/*
+	 * If any completed reads were found and we added buffers, advance
+	 * the buffer ring and prepare a single bundle send for all of them.
+	 */
+	if (first_req) {
+		io_uring_buf_ring_advance(br, nr);
+
+		sqe = io_uring_get_sqe(&kdigest->ring);
+		io_uring_prep_send_bundle(sqe, outfd, 0, MSG_MORE);
+		sqe->flags |= IOSQE_BUFFER_SELECT;
+		sqe->buf_group = BGID;
+		io_uring_sqe_set_data(sqe, first_req);
+		(*inflight)++;
+	}
+}
+
+/*
+ * Serialize multiple writes with IOSQE_IO_LINK. Not the most efficient
+ * way, as it's both more expensive on the kernel side to handle link, and
+ * if there's bundle support, all of the below can be done with a single
+ * send rather than multiple ones.
+ */
+static void submit_sends_linked(struct kdigest *kdigest, int *write_idx,
+				int *inflight)
+{
+	struct io_uring_sqe *sqe;
+	struct req *req;
+
+	/* Queue up any possible writes. Link flag ensures ordering. */
+	sqe = NULL;
+	while (kdigest->reqs[*write_idx].state == IO_READ_COMPLETE) {
+		if (sqe)
+			sqe->flags |= IOSQE_IO_LINK;
+
+		req = &kdigest->reqs[*write_idx];
+		req->state = IO_WRITE;
+		sqe = io_uring_get_sqe(&kdigest->ring);
+		io_uring_prep_send(sqe, outfd, req->iov.iov_base,
+					req->iov.iov_len, MSG_MORE);
+		io_uring_sqe_set_data(sqe, req);
+		(*inflight)++;
+
+		*write_idx = (*write_idx + 1) % QD;
+	}
+}
+
+static void submit_sends(struct kdigest *kdigest, int *write_idx, int *inflight)
+{
+	if (kdigest->br)
+		submit_sends_br(kdigest, write_idx, inflight);
+	else
+		submit_sends_linked(kdigest, write_idx, inflight);
+}
+
+static int digest_file(struct kdigest *kdigest, size_t insize)
+{
+	struct io_uring *ring = &kdigest->ring;
+	off_t read_off = 0;
+	size_t outsize = insize;
+	int read_idx = 0, write_idx = 0, inflight = 0;
+
+	while (outsize) {
+		struct io_uring_sqe *sqe;
+		struct req *req;
+		int to_wait;
+
+		submit_sends(kdigest, &write_idx, &inflight);
+
+		/* Queue up any reads. Completions may arrive out of order. */
+		while (insize && (kdigest->reqs[read_idx].state == IO_INIT
+		    || kdigest->reqs[read_idx].state == IO_WRITE_COMPLETE)) {
+			size_t this_size = (insize < BS ? insize : BS);
+
+			req = &kdigest->reqs[read_idx];
+			req->state = IO_READ;
+			req->offset = read_off;
+			req->iov.iov_base = &kdigest->bufs[read_idx * BS];
+			req->iov.iov_len = this_size;
+
+			sqe = io_uring_get_sqe(ring);
+			io_uring_prep_read(sqe, infd, req->iov.iov_base,
+						req->iov.iov_len, read_off);
+			io_uring_sqe_set_data(sqe, req);
+
+			read_off += this_size;
+			insize -= this_size;
+			inflight++;
+
+			read_idx = (read_idx + 1) % QD;
+		}
+
+		/* wait for about half queue completion before resubmit */
+		for (to_wait = (inflight >> 1) | 1; to_wait; to_wait--) {
+			int ret, wait_nr;
+
+			wait_nr = inflight;
+			if (wait_nr > WAIT_BATCH)
+				wait_nr = WAIT_BATCH;
+
+			ret = io_uring_submit_and_wait(ring, wait_nr);
+			if (ret < 0) {
+				fprintf(stderr, "wait cqe: %s\n",
+					strerror(-ret));
+				return 1;
+			}
+
+			if (reap_completions(ring, &inflight, &outsize))
+				return 1;
+		}
+	}
+	assert(!inflight);
+
+	return 0;
+}
+
+static int get_result(struct kdigest *kdigest, const char *alg, const char *file)
+{
+	struct io_uring *ring = &kdigest->ring;
+	struct io_uring_sqe *sqe;
+	struct io_uring_cqe *cqe;
+	int i, ret;
+	/* reuse I/O buf block to stash hash result */
+
+	sqe = io_uring_get_sqe(ring);
+	io_uring_prep_recv(sqe, outfd, kdigest->bufs, BS, 0);
+
+	if (io_uring_submit_and_wait(ring, 1) < 0)
+		return 1;
+
+	ret = io_uring_peek_cqe(ring, &cqe);
+	if (ret < 0) {
+		fprintf(stderr, "peek cqe: %s\n", strerror(-ret));
+		return 1;
+	}
+
+	if (cqe->res < 0) {
+		fprintf(stderr, "cqe error: %s\n", strerror(-cqe->res));
+		goto err;
+	}
+
+	fprintf(stdout, "uring %s%s(%s) returned(len=%u): ",
+		kdigest->br ? "bundled " : "", alg, file, cqe->res);
+	for (i = 0; i < cqe->res; i++)
+		fprintf(stdout, "%02x", kdigest->bufs[i]);
+	putc('\n', stdout);
+	ret = 0;
+err:
+	io_uring_cqe_seen(ring, cqe);
+	return ret;
+}
+
+int main(int argc, char *argv[])
+{
+	const char *alg;
+	const char *infile;
+	size_t alg_len, insize;
+	struct sockaddr_alg sa = {
+		.salg_family = AF_ALG,
+		.salg_type = "hash",
+	};
+	struct kdigest kdigest = { };
+	struct io_uring_params p = { };
+	int sfd, ret;
+
+	if (argc < 3) {
+		fprintf(stderr, "%s: algorithm infile\n", argv[0]);
+		return 1;
+	}
+
+	alg = argv[1];
+	infile = argv[2];
+	alg_len = strlen(alg);
+	if (alg_len >= sizeof(sa.salg_name)) {
+		fprintf(stderr, "algorithm name too long\n");
+		return 1;
+	}
+	/* +1 for null terminator */
+	memcpy(sa.salg_name, alg, alg_len + 1);
+
+	infd = open(infile, O_RDONLY);
+	if (infd < 0) {
+		perror("open infile");
+		return 1;
+	}
+
+	sfd = socket(AF_ALG, SOCK_SEQPACKET, 0);
+	if (sfd < 0) {
+		if (errno == EAFNOSUPPORT)
+			fprintf(stderr, "kernel AF_ALG support not available. "
+				"CONFIG_CRYPTO_USER_API_HASH required.\n");
+		else
+			perror("AF_ALG socket");
+		return 1;
+	}
+
+	if (bind(sfd, (struct sockaddr *)&sa, sizeof(sa)) < 0) {
+		if (errno == ENOENT)
+			fprintf(stderr, "AF_ALG bind(%s): hash not available. "
+				"See /proc/crypto hash algorithm list.\n",
+				alg);
+		else
+			fprintf(stderr, "AF_ALG bind(%s): %s\n",
+				alg, strerror(errno));
+		return 1;
+	}
+
+	outfd = accept(sfd, NULL, 0);
+	if (outfd < 0) {
+		perror("AF_ALG accept");
+		return 1;
+	}
+
+	if (posix_memalign((void **)&kdigest.bufs, 4096, QD * BS)) {
+		fprintf(stderr, "failed to alloc I/O bufs\n");
+		return 1;
+	}
+
+	p.flags = IORING_SETUP_SINGLE_ISSUER | IORING_SETUP_DEFER_TASKRUN;
+	do {
+		ret = io_uring_queue_init_params(QD, &kdigest.ring, &p);
+		if (!ret)
+			break;
+		if (!p.flags) {
+			fprintf(stderr, "queue_init: %s\n", strerror(-ret));
+			return 1;
+		}
+		p.flags = 0;
+	} while (1);
+
+	/* use send bundles, if available */
+	if (p.features & IORING_FEAT_RECVSEND_BUNDLE) {
+		kdigest.br = io_uring_setup_buf_ring(&kdigest.ring, QD, BGID, 0, &ret);
+		if (!kdigest.br) {
+			fprintf(stderr, "Failed setting up bundle buffer ring: %d\n", ret);
+			return 1;
+		}
+	}
+
+	if (get_file_size(infd, &insize))
+		return 1;
+
+	ret = digest_file(&kdigest, insize);
+	if (ret) {
+		fprintf(stderr, "%s digest failed\n", alg);
+		return 1;
+	}
+
+	ret = get_result(&kdigest, alg, infile);
+	if (ret) {
+		fprintf(stderr, "failed to retrieve %s digest result\n", alg);
+		return 1;
+	}
+
+	if (kdigest.br)
+		io_uring_free_buf_ring(&kdigest.ring, kdigest.br, QD, BGID);
+	io_uring_queue_exit(&kdigest.ring);
+	free(kdigest.bufs);
+	if (close(infd) < 0)
+		ret |= 1;
+	if (close(sfd) < 0)
+		ret |= 1;
+	if (close(outfd) < 0)
+		ret |= 1;
+	return ret;
+}

data/vendor/liburing/examples/proxy.c

@@ -82,6 +82,7 @@ static char *host = "192.168.3.2";
 static int send_port = 4445;
 static int receive_port = 4444;
 static int buf_size = 32;
+static int buf_ring_inc;
 static int bidi;
 static int ipv6;
 static int napi;
@@ -356,6 +357,7 @@ static void free_buffer_rings(struct io_uring *ring, struct conn *c)
 static int setup_recv_ring(struct io_uring *ring, struct conn *c)
 {
 	struct conn_buf_ring *cbr = &c->in_br;
+	int br_flags = 0;
 	int ret, i;
 	size_t len;
 	void *ptr;
@@ -375,7 +377,9 @@ static int setup_recv_ring(struct io_uring *ring, struct conn *c)
 			return 1;
 		}
 	}
-	cbr->br = io_uring_setup_buf_ring(ring, nr_bufs, cbr->bgid, 0, &ret);
+	if (buf_ring_inc)
+		br_flags = IOU_PBUF_RING_INC;
+	cbr->br = io_uring_setup_buf_ring(ring, nr_bufs, cbr->bgid, br_flags, &ret);
 	if (!cbr->br) {
 		fprintf(stderr, "Buffer ring register failed %d\n", ret);
 		return 1;
@@ -401,9 +405,12 @@ static int setup_recv_ring(struct io_uring *ring, struct conn *c)
 static int setup_send_ring(struct io_uring *ring, struct conn *c)
 {
 	struct conn_buf_ring *cbr = &c->out_br;
+	int br_flags = 0;
 	int ret;
 
-	cbr->br = io_uring_setup_buf_ring(ring, nr_bufs, cbr->bgid, 0, &ret);
+	if (buf_ring_inc)
+		br_flags = IOU_PBUF_RING_INC;
+	cbr->br = io_uring_setup_buf_ring(ring, nr_bufs, cbr->bgid, br_flags, &ret);
 	if (!cbr->br) {
 		fprintf(stderr, "Buffer ring register failed %d\n", ret);
 		return 1;
@@ -1148,6 +1155,32 @@ static int recv_done_res(int res)
 	return 0;
 }
 
+static int recv_inc(struct conn *c, struct conn_dir *cd, int *bid,
+		    struct io_uring_cqe *cqe)
+{
+	struct conn_buf_ring *cbr = &c->out_br;
+	struct conn_buf_ring *in_cbr = &c->in_br;
+	void *data;
+
+	if (!cqe->res)
+		return 0;
+	if (cqe->flags & IORING_CQE_F_BUF_MORE)
+		return 0;
+
+	data = in_cbr->buf + *bid * buf_size;
+	if (is_sink) {
+		io_uring_buf_ring_add(in_cbr->br, data, buf_size, *bid, br_mask, 0);
+		io_uring_buf_ring_advance(in_cbr->br, 1);
+	} else if (send_ring) {
+		io_uring_buf_ring_add(cbr->br, data, buf_size, *bid, br_mask, 0);
+		io_uring_buf_ring_advance(cbr->br, 1);
+	} else {
+		send_append(c, cd, data, *bid, buf_size);
+	}
+	*bid = (*bid + 1) & (nr_bufs - 1);
+	return 1;
+}
+
 /*
  * Any receive that isn't recvmsg with multishot can be handled the same way.
  * Iterate from '*bid' and 'in_bytes' in total, and append the data to the
@@ -1291,7 +1324,9 @@ start_close:
 	 * end and the buffer will be replenished once the send is done with
 	 * it.
 	 */
-	if (is_sink)
+	if (buf_ring_inc)
+		nr_packets = recv_inc(c, ocd, &bid, cqe);
+	else if (is_sink)
 		nr_packets = replenish_buffers(c, &bid, cqe->res);
 	else if (rcv_msg && recv_mshot)
 		nr_packets = recv_mshot_msg(c, ocd, &bid, cqe->res);
@@ -1318,7 +1353,7 @@ start_close:
 		cd->pending_recv = 0;
 		if (recv_done_res(cqe->res))
 			goto start_close;
-		if (is_sink)
+		if (is_sink || !ocd->pending_send)
 			__submit_receive(ring, c, &c->cd[0], c->in_fd);
 	}
 
@@ -1480,12 +1515,39 @@ static int prep_next_send(struct io_uring *ring, struct conn *c,
 	}
 }
 
+static int handle_send_inc(struct conn *c, struct conn_dir *cd, int bid,
+			   struct io_uring_cqe *cqe)
+{
+	struct conn_buf_ring *in_cbr = &c->in_br;
+	int ret = 0;
+	void *data;
+
+	if (!cqe->res)
+		goto out;
+	if (cqe->flags & IORING_CQE_F_BUF_MORE)
+		return 0;
+
+	assert(cqe->res <= buf_size);
+	cd->out_bytes += cqe->res;
+
+	data = in_cbr->buf + bid * buf_size;
+	io_uring_buf_ring_add(in_cbr->br, data, buf_size, bid, br_mask, 0);
+	io_uring_buf_ring_advance(in_cbr->br, 1);
+	bid = (bid + 1) & (nr_bufs - 1);
+	ret = 1;
+out:
+	if (pending_shutdown(c))
+		close_cd(c, cd);
+
+	return ret;
+}
+
 /*
  * Handling a send with an outgoing send ring. Get the buffers from the
  * receive side, and add them to the ingoing buffer ring again.
  */
-static int handle_send_ring(struct conn *c, struct conn_dir *cd,
-			    int bid, int bytes)
+static int handle_send_ring(struct conn *c, struct conn_dir *cd, int bid,
+			    int bytes)
 {
 	struct conn_buf_ring *in_cbr = &c->in_br;
 	struct conn_buf_ring *out_cbr = &c->out_br;
@@ -1605,7 +1667,9 @@ static int __handle_send(struct io_uring *ring, struct conn *c,
 
 		vlog("send: got %d, %lu\n", cqe->res, cd->out_bytes);
 
-		if (send_ring)
+		if (buf_ring_inc)
+			nr_packets = handle_send_inc(c, cd, bid, cqe);
+		else if (send_ring)
 			nr_packets = handle_send_ring(c, cd, bid, cqe->res);
 		else
 			nr_packets = handle_send_buf(c, cd, bid, cqe->res);
@@ -2305,7 +2369,7 @@ int main(int argc, char *argv[])
 
 	pthread_mutex_init(&thread_lock, NULL);
 
-	optstring = "m:d:S:s:b:f:H:r:p:n:B:N:T:w:t:M:R:u:c:C:q:a:x:z:6Vh?";
+	optstring = "m:d:S:s:b:f:H:r:p:n:B:N:T:w:t:M:R:u:c:C:q:a:x:z:i:6Vh?";
 	while ((opt = getopt(argc, argv, optstring)) != -1) {
 		switch (opt) {
 		case 'm':
@@ -2377,6 +2441,9 @@ int main(int argc, char *argv[])
 		case 'q':
 			ring_size = atoi(optarg);
 			break;
+		case 'i':
+			buf_ring_inc = !!atoi(optarg);
+			break;
 		case 'a':
 			use_huge = !!atoi(optarg);
 			break;

data/vendor/liburing/liburing.pc.in

@@ -9,4 +9,4 @@ Description: io_uring library
 URL: https://git.kernel.dk/cgit/liburing/
 
 Libs: -L${libdir} -luring
-Cflags: -I${includedir} -D_GNU_SOURCE
+Cflags: -I${includedir}

data/vendor/liburing/src/Makefile

@@ -16,8 +16,6 @@ override CFLAGS += -Wno-unused-parameter \
 	$(LIBURING_CFLAGS)
 SO_CFLAGS=-fPIC $(CFLAGS)
 L_CFLAGS=$(CFLAGS)
-LINK_FLAGS=-Wl,-z,defs
-LINK_FLAGS+=$(LDFLAGS)
 ENABLE_SHARED ?= 1
 
 soname=liburing.so.$(VERSION_MAJOR)
@@ -39,6 +37,13 @@ ifneq ($(MAKECMDGOALS),clean)
 include ../config-host.mak
 endif
 
+ifeq ($(CONFIG_USE_SANITIZER),y)
+	LINK_FLAGS=
+else
+	LINK_FLAGS=-Wl,-z,defs
+endif
+LINK_FLAGS+=$(LDFLAGS)
+
 all: $(all_targets)
 
 liburing_srcs := setup.c queue.c register.c syscall.c version.c
@@ -50,6 +55,13 @@ ifeq ($(CONFIG_NOLIBC),y)
 	override LINK_FLAGS += -nostdlib -nodefaultlibs $(libgcc_link_flag)
 endif
 
+ifeq ($(CONFIG_USE_SANITIZER),y)
+	override CFLAGS += -fsanitize=address,undefined -g -fno-omit-frame-pointer -fno-optimize-sibling-calls
+	override CPPFLAGS += -fsanitize=address,undefined -g -fno-omit-frame-pointer -fno-optimize-sibling-calls
+	override LINK_FLAGS += -fsanitize=address,undefined
+	liburing_srcs += sanitize.c
+endif
+
 override CPPFLAGS += -MT "$@" -MMD -MP -MF "$@.d"
 liburing_objs := $(patsubst %.c,%.ol,$(liburing_srcs))
 liburing_sobjs := $(patsubst %.c,%.os,$(liburing_srcs))
@@ -89,6 +101,7 @@ install: $(all_targets)
 	install -D -m 644 include/liburing.h $(includedir)/liburing.h
 	install -D -m 644 include/liburing/compat.h $(includedir)/liburing/compat.h
 	install -D -m 644 include/liburing/barrier.h $(includedir)/liburing/barrier.h
+	install -D -m 644 include/liburing/sanitize.h $(includedir)/liburing/sanitize.h
 	install -D -m 644 include/liburing/io_uring_version.h $(includedir)/liburing/io_uring_version.h
 	install -D -m 644 liburing.a $(libdevdir)/liburing.a
 	install -D -m 644 liburing-ffi.a $(libdevdir)/liburing-ffi.a
@@ -106,6 +119,7 @@ uninstall:
 	@rm -f $(includedir)/liburing.h
 	@rm -f $(includedir)/liburing/compat.h
 	@rm -f $(includedir)/liburing/barrier.h
+	@rm -f $(includedir)/liburing/sanitize.h
 	@rm -f $(includedir)/liburing/io_uring_version.h
 	@rm -f $(libdevdir)/liburing.a
 	@rm -f $(libdevdir)/liburing-ffi.a

data/vendor/liburing/src/include/liburing/io_uring.h

@@ -440,11 +440,21 @@ struct io_uring_cqe {
  * IORING_CQE_F_SOCK_NONEMPTY	If set, more data to read after socket recv
  * IORING_CQE_F_NOTIF	Set for notification CQEs. Can be used to distinct
  * 			them from sends.
+ * IORING_CQE_F_BUF_MORE If set, the buffer ID set in the completion will get
+ *			more completions. In other words, the buffer is being
+ *			partially consumed, and will be used by the kernel for
+ *			more completions. This is only set for buffers used via
+ *			the incremental buffer consumption, as provided by
+ *			a ring buffer setup with IOU_PBUF_RING_INC. For any
+ *			other provided buffer type, all completions with a
+ *			buffer passed back is automatically returned to the
+ *			application.
  */
 #define IORING_CQE_F_BUFFER		(1U << 0)
 #define IORING_CQE_F_MORE		(1U << 1)
 #define IORING_CQE_F_SOCK_NONEMPTY	(1U << 2)
 #define IORING_CQE_F_NOTIF		(1U << 3)
+#define IORING_CQE_F_BUF_MORE		(1U << 4)
 
 #define IORING_CQE_BUFFER_SHIFT		16
 
@@ -599,6 +609,9 @@ enum io_uring_register_op {
 
 	IORING_REGISTER_CLOCK			= 29,
 
+	/* clone registered buffers from source ring to current ring */
+	IORING_REGISTER_CLONE_BUFFERS		= 30,
+
 	/* this goes last */
 	IORING_REGISTER_LAST,
 
@@ -684,6 +697,16 @@ struct io_uring_clock_register {
 	__u32	__resv[3];
 };
 
+enum {
+	IORING_REGISTER_SRC_REGISTERED = 1,
+};
+
+struct io_uring_clone_buffers {
+	__u32	src_fd;
+	__u32	flags;
+	__u32	pad[6];
+};
+
 struct io_uring_buf {
 	__u64	addr;
 	__u32	len;
@@ -716,9 +739,17 @@ struct io_uring_buf_ring {
  *			mmap(2) with the offset set as:
  *			IORING_OFF_PBUF_RING | (bgid << IORING_OFF_PBUF_SHIFT)
  *			to get a virtual mapping for the ring.
+ * IOU_PBUF_RING_INC:	If set, buffers consumed from this buffer ring can be
+ *			consumed incrementally. Normally one (or more) buffers
+ *			are fully consumed. With incremental consumptions, it's
+ *			feasible to register big ranges of buffers, and each
+ *			use of it will consume only as much as it needs. This
+ *			requires that both the kernel and application keep
+ *			track of where the current read/recv index is at.
  */
 enum io_uring_register_pbuf_ring_flags {
 	IOU_PBUF_RING_MMAP	= 1,
+	IOU_PBUF_RING_INC	= 2,
 };
 
 /* argument for IORING_(UN)REGISTER_PBUF_RING */