uringmachine 0.3 → 0.5

data/vendor/liburing/examples/kdigest.c

@@ -0,0 +1,405 @@
+/* SPDX-License-Identifier: MIT */
+/*
+ * Proof-of-concept for doing file digests using the kernel's AF_ALG API.
+ * Needs a bit of error handling.
+ */
+#include <stdio.h>
+#include <fcntl.h>
+#include <string.h>
+#include <stdlib.h>
+#include <unistd.h>
+#include <assert.h>
+#include <errno.h>
+#include <inttypes.h>
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <sys/ioctl.h>
+#include <linux/if_alg.h>
+#include "liburing.h"
+
+#define QD		64
+#define WAIT_BATCH	(QD / 8)
+#define BS		(64*1024)
+
+#define BGID		1
+#define BID_MASK	(QD - 1)
+
+enum req_state {
+	IO_INIT = 0,
+	IO_READ,
+	IO_READ_COMPLETE,
+	IO_WRITE,
+	IO_WRITE_COMPLETE,
+};
+
+struct req {
+	off_t offset;
+	enum req_state state;
+	struct iovec iov;
+};
+
+struct kdigest {
+	struct io_uring ring;
+	struct io_uring_buf_ring *br;
+	struct req reqs[QD];
+	/* heap allocated, aligned QD*BS buffer */
+	uint8_t *bufs;
+};
+
+static int infd, outfd;
+
+static int get_file_size(int fd, size_t *size)
+{
+	struct stat st;
+
+	if (fstat(fd, &st) < 0)
+		return -1;
+	if (S_ISREG(st.st_mode)) {
+		*size = st.st_size;
+	} else if (S_ISBLK(st.st_mode)) {
+		unsigned long long bytes;
+
+		if (ioctl(fd, BLKGETSIZE64, &bytes) != 0)
+			return -1;
+
+		*size = bytes;
+	} else {
+		return -1;
+	}
+
+	return 0;
+}
+
+static int reap_completions(struct io_uring *ring, int *inflight,
+			    size_t *outsize)
+{
+	struct io_uring_cqe *cqe;
+	unsigned head;
+	int ret = 0, nr;
+
+	nr = 0;
+	io_uring_for_each_cqe(ring, head, cqe) {
+		struct req *req;
+
+		req = io_uring_cqe_get_data(cqe);
+		assert(req->state == IO_READ || req->state == IO_WRITE);
+		if (cqe->res < 0) {
+			fprintf(stderr, "%s: cqe error %d\n",
+				req->state == IO_WRITE ? "send" : "read",
+				cqe->res);
+			*outsize = 0;
+			ret = 1;
+			break;
+		}
+
+		(*inflight)--;
+		req->state++;
+		if (req->state == IO_WRITE_COMPLETE)
+			*outsize -= cqe->res;
+		nr++;
+	}
+
+	io_uring_cq_advance(ring, nr);
+	return ret;
+}
+
+/*
+ * Add buffers to the outgoing ring, and submit a single bundle send that
+ * will finish when all of them have completed.
+ */
+static void submit_sends_br(struct kdigest *kdigest, int *write_idx,
+			    int *inflight)
+{
+	struct io_uring_buf_ring *br = kdigest->br;
+	struct req *req, *first_req = NULL;
+	struct io_uring_sqe *sqe;
+	int nr = 0;
+
+	/*
+	 * Find any completed reads, and add the buffers to the outgoing
+	 * send ring. That will serialize the data sent.
+	 */
+	while (kdigest->reqs[*write_idx].state == IO_READ_COMPLETE) {
+		req = &kdigest->reqs[*write_idx];
+		io_uring_buf_ring_add(br, req->iov.iov_base, req->iov.iov_len,
+					*write_idx, BID_MASK, nr++);
+		/*
+		 * Mark as a write/send if it's the first one, that serve
+		 * as the "barrier" in the array. The rest can be marked
+		 * complete upfront, if there's more in this bundle, as
+		 * the first will serve a the stopping point.
+		 */
+		if (!first_req) {
+			req->state = IO_WRITE;
+			first_req = req;
+		} else {
+			req->state = IO_WRITE_COMPLETE;
+		}
+		*write_idx = (*write_idx + 1) % QD;
+	}
+
+	/*
+	 * If any completed reads were found and we added buffers, advance
+	 * the buffer ring and prepare a single bundle send for all of them.
+	 */
+	if (first_req) {
+		io_uring_buf_ring_advance(br, nr);
+
+		sqe = io_uring_get_sqe(&kdigest->ring);
+		io_uring_prep_send_bundle(sqe, outfd, 0, MSG_MORE);
+		sqe->flags |= IOSQE_BUFFER_SELECT;
+		sqe->buf_group = BGID;
+		io_uring_sqe_set_data(sqe, first_req);
+		(*inflight)++;
+	}
+}
+
+/*
+ * Serialize multiple writes with IOSQE_IO_LINK. Not the most efficient
+ * way, as it's both more expensive on the kernel side to handle link, and
+ * if there's bundle support, all of the below can be done with a single
+ * send rather than multiple ones.
+ */
+static void submit_sends_linked(struct kdigest *kdigest, int *write_idx,
+				int *inflight)
+{
+	struct io_uring_sqe *sqe;
+	struct req *req;
+
+	/* Queue up any possible writes. Link flag ensures ordering. */
+	sqe = NULL;
+	while (kdigest->reqs[*write_idx].state == IO_READ_COMPLETE) {
+		if (sqe)
+			sqe->flags |= IOSQE_IO_LINK;
+
+		req = &kdigest->reqs[*write_idx];
+		req->state = IO_WRITE;
+		sqe = io_uring_get_sqe(&kdigest->ring);
+		io_uring_prep_send(sqe, outfd, req->iov.iov_base,
+					req->iov.iov_len, MSG_MORE);
+		io_uring_sqe_set_data(sqe, req);
+		(*inflight)++;
+
+		*write_idx = (*write_idx + 1) % QD;
+	}
+}
+
+static void submit_sends(struct kdigest *kdigest, int *write_idx, int *inflight)
+{
+	if (kdigest->br)
+		submit_sends_br(kdigest, write_idx, inflight);
+	else
+		submit_sends_linked(kdigest, write_idx, inflight);
+}
+
+static int digest_file(struct kdigest *kdigest, size_t insize)
+{
+	struct io_uring *ring = &kdigest->ring;
+	off_t read_off = 0;
+	size_t outsize = insize;
+	int read_idx = 0, write_idx = 0, inflight = 0;
+
+	while (outsize) {
+		struct io_uring_sqe *sqe;
+		struct req *req;
+		int to_wait;
+
+		submit_sends(kdigest, &write_idx, &inflight);
+
+		/* Queue up any reads. Completions may arrive out of order. */
+		while (insize && (kdigest->reqs[read_idx].state == IO_INIT
+		    || kdigest->reqs[read_idx].state == IO_WRITE_COMPLETE)) {
+			size_t this_size = (insize < BS ? insize : BS);
+
+			req = &kdigest->reqs[read_idx];
+			req->state = IO_READ;
+			req->offset = read_off;
+			req->iov.iov_base = &kdigest->bufs[read_idx * BS];
+			req->iov.iov_len = this_size;
+
+			sqe = io_uring_get_sqe(ring);
+			io_uring_prep_read(sqe, infd, req->iov.iov_base,
+						req->iov.iov_len, read_off);
+			io_uring_sqe_set_data(sqe, req);
+
+			read_off += this_size;
+			insize -= this_size;
+			inflight++;
+
+			read_idx = (read_idx + 1) % QD;
+		}
+
+		/* wait for about half queue completion before resubmit */
+		for (to_wait = (inflight >> 1) | 1; to_wait; to_wait--) {
+			int ret, wait_nr;
+
+			wait_nr = inflight;
+			if (wait_nr > WAIT_BATCH)
+				wait_nr = WAIT_BATCH;
+
+			ret = io_uring_submit_and_wait(ring, wait_nr);
+			if (ret < 0) {
+				fprintf(stderr, "wait cqe: %s\n",
+					strerror(-ret));
+				return 1;
+			}
+
+			if (reap_completions(ring, &inflight, &outsize))
+				return 1;
+		}
+	}
+	assert(!inflight);
+
+	return 0;
+}
+
+static int get_result(struct kdigest *kdigest, const char *alg, const char *file)
+{
+	struct io_uring *ring = &kdigest->ring;
+	struct io_uring_sqe *sqe;
+	struct io_uring_cqe *cqe;
+	int i, ret;
+	/* reuse I/O buf block to stash hash result */
+
+	sqe = io_uring_get_sqe(ring);
+	io_uring_prep_recv(sqe, outfd, kdigest->bufs, BS, 0);
+
+	if (io_uring_submit_and_wait(ring, 1) < 0)
+		return 1;
+
+	ret = io_uring_peek_cqe(ring, &cqe);
+	if (ret < 0) {
+		fprintf(stderr, "peek cqe: %s\n", strerror(-ret));
+		return 1;
+	}
+
+	if (cqe->res < 0) {
+		fprintf(stderr, "cqe error: %s\n", strerror(-cqe->res));
+		goto err;
+	}
+
+	fprintf(stdout, "uring %s%s(%s) returned(len=%u): ",
+		kdigest->br ? "bundled " : "", alg, file, cqe->res);
+	for (i = 0; i < cqe->res; i++)
+		fprintf(stdout, "%02x", kdigest->bufs[i]);
+	putc('\n', stdout);
+	ret = 0;
+err:
+	io_uring_cqe_seen(ring, cqe);
+	return ret;
+}
+
+int main(int argc, char *argv[])
+{
+	const char *alg;
+	const char *infile;
+	size_t alg_len, insize;
+	struct sockaddr_alg sa = {
+		.salg_family = AF_ALG,
+		.salg_type = "hash",
+	};
+	struct kdigest kdigest = { };
+	struct io_uring_params p = { };
+	int sfd, ret;
+
+	if (argc < 3) {
+		fprintf(stderr, "%s: algorithm infile\n", argv[0]);
+		return 1;
+	}
+
+	alg = argv[1];
+	infile = argv[2];
+	alg_len = strlen(alg);
+	if (alg_len >= sizeof(sa.salg_name)) {
+		fprintf(stderr, "algorithm name too long\n");
+		return 1;
+	}
+	/* +1 for null terminator */
+	memcpy(sa.salg_name, alg, alg_len + 1);
+
+	infd = open(infile, O_RDONLY);
+	if (infd < 0) {
+		perror("open infile");
+		return 1;
+	}
+
+	sfd = socket(AF_ALG, SOCK_SEQPACKET, 0);
+	if (sfd < 0) {
+		if (errno == EAFNOSUPPORT)
+			fprintf(stderr, "kernel AF_ALG support not available. "
+				"CONFIG_CRYPTO_USER_API_HASH required.\n");
+		else
+			perror("AF_ALG socket");
+		return 1;
+	}
+
+	if (bind(sfd, (struct sockaddr *)&sa, sizeof(sa)) < 0) {
+		if (errno == ENOENT)
+			fprintf(stderr, "AF_ALG bind(%s): hash not available. "
+				"See /proc/crypto hash algorithm list.\n",
+				alg);
+		else
+			fprintf(stderr, "AF_ALG bind(%s): %s\n",
+				alg, strerror(errno));
+		return 1;
+	}
+
+	outfd = accept(sfd, NULL, 0);
+	if (outfd < 0) {
+		perror("AF_ALG accept");
+		return 1;
+	}
+
+	if (posix_memalign((void **)&kdigest.bufs, 4096, QD * BS)) {
+		fprintf(stderr, "failed to alloc I/O bufs\n");
+		return 1;
+	}
+
+	p.flags = IORING_SETUP_SINGLE_ISSUER | IORING_SETUP_DEFER_TASKRUN;
+	do {
+		ret = io_uring_queue_init_params(QD, &kdigest.ring, &p);
+		if (!ret)
+			break;
+		if (!p.flags) {
+			fprintf(stderr, "queue_init: %s\n", strerror(-ret));
+			return 1;
+		}
+		p.flags = 0;
+	} while (1);
+
+	/* use send bundles, if available */
+	if (p.features & IORING_FEAT_RECVSEND_BUNDLE) {
+		kdigest.br = io_uring_setup_buf_ring(&kdigest.ring, QD, BGID, 0, &ret);
+		if (!kdigest.br) {
+			fprintf(stderr, "Failed setting up bundle buffer ring: %d\n", ret);
+			return 1;
+		}
+	}
+
+	if (get_file_size(infd, &insize))
+		return 1;
+
+	ret = digest_file(&kdigest, insize);
+	if (ret) {
+		fprintf(stderr, "%s digest failed\n", alg);
+		return 1;
+	}
+
+	ret = get_result(&kdigest, alg, infile);
+	if (ret) {
+		fprintf(stderr, "failed to retrieve %s digest result\n", alg);
+		return 1;
+	}
+
+	if (kdigest.br)
+		io_uring_free_buf_ring(&kdigest.ring, kdigest.br, QD, BGID);
+	io_uring_queue_exit(&kdigest.ring);
+	free(kdigest.bufs);
+	if (close(infd) < 0)
+		ret |= 1;
+	if (close(sfd) < 0)
+		ret |= 1;
+	if (close(outfd) < 0)
+		ret |= 1;
+	return ret;
+}

data/vendor/liburing/examples/proxy.c

@@ -82,6 +82,7 @@ static char *host = "192.168.3.2";
 static int send_port = 4445;
 static int receive_port = 4444;
 static int buf_size = 32;
+static int buf_ring_inc;
 static int bidi;
 static int ipv6;
 static int napi;
@@ -356,6 +357,7 @@ static void free_buffer_rings(struct io_uring *ring, struct conn *c)
 static int setup_recv_ring(struct io_uring *ring, struct conn *c)
 {
 	struct conn_buf_ring *cbr = &c->in_br;
+	int br_flags = 0;
 	int ret, i;
 	size_t len;
 	void *ptr;
@@ -375,7 +377,9 @@ static int setup_recv_ring(struct io_uring *ring, struct conn *c)
 			return 1;
 		}
 	}
-	cbr->br = io_uring_setup_buf_ring(ring, nr_bufs, cbr->bgid, 0, &ret);
+	if (buf_ring_inc)
+		br_flags = IOU_PBUF_RING_INC;
+	cbr->br = io_uring_setup_buf_ring(ring, nr_bufs, cbr->bgid, br_flags, &ret);
 	if (!cbr->br) {
 		fprintf(stderr, "Buffer ring register failed %d\n", ret);
 		return 1;
@@ -401,9 +405,12 @@ static int setup_recv_ring(struct io_uring *ring, struct conn *c)
 static int setup_send_ring(struct io_uring *ring, struct conn *c)
 {
 	struct conn_buf_ring *cbr = &c->out_br;
+	int br_flags = 0;
 	int ret;
 
-	cbr->br = io_uring_setup_buf_ring(ring, nr_bufs, cbr->bgid, 0, &ret);
+	if (buf_ring_inc)
+		br_flags = IOU_PBUF_RING_INC;
+	cbr->br = io_uring_setup_buf_ring(ring, nr_bufs, cbr->bgid, br_flags, &ret);
 	if (!cbr->br) {
 		fprintf(stderr, "Buffer ring register failed %d\n", ret);
 		return 1;
@@ -1148,6 +1155,32 @@ static int recv_done_res(int res)
 	return 0;
 }
 
+static int recv_inc(struct conn *c, struct conn_dir *cd, int *bid,
+		    struct io_uring_cqe *cqe)
+{
+	struct conn_buf_ring *cbr = &c->out_br;
+	struct conn_buf_ring *in_cbr = &c->in_br;
+	void *data;
+
+	if (!cqe->res)
+		return 0;
+	if (cqe->flags & IORING_CQE_F_BUF_MORE)
+		return 0;
+
+	data = in_cbr->buf + *bid * buf_size;
+	if (is_sink) {
+		io_uring_buf_ring_add(in_cbr->br, data, buf_size, *bid, br_mask, 0);
+		io_uring_buf_ring_advance(in_cbr->br, 1);
+	} else if (send_ring) {
+		io_uring_buf_ring_add(cbr->br, data, buf_size, *bid, br_mask, 0);
+		io_uring_buf_ring_advance(cbr->br, 1);
+	} else {
+		send_append(c, cd, data, *bid, buf_size);
+	}
+	*bid = (*bid + 1) & (nr_bufs - 1);
+	return 1;
+}
+
 /*
  * Any receive that isn't recvmsg with multishot can be handled the same way.
  * Iterate from '*bid' and 'in_bytes' in total, and append the data to the
@@ -1291,7 +1324,9 @@ start_close:
 	 * end and the buffer will be replenished once the send is done with
 	 * it.
 	 */
-	if (is_sink)
+	if (buf_ring_inc)
+		nr_packets = recv_inc(c, ocd, &bid, cqe);
+	else if (is_sink)
 		nr_packets = replenish_buffers(c, &bid, cqe->res);
 	else if (rcv_msg && recv_mshot)
 		nr_packets = recv_mshot_msg(c, ocd, &bid, cqe->res);
@@ -1318,7 +1353,7 @@ start_close:
 		cd->pending_recv = 0;
 		if (recv_done_res(cqe->res))
 			goto start_close;
-		if (is_sink)
+		if (is_sink || !ocd->pending_send)
 			__submit_receive(ring, c, &c->cd[0], c->in_fd);
 	}
 
@@ -1480,12 +1515,39 @@ static int prep_next_send(struct io_uring *ring, struct conn *c,
 	}
 }
 
+static int handle_send_inc(struct conn *c, struct conn_dir *cd, int bid,
+			   struct io_uring_cqe *cqe)
+{
+	struct conn_buf_ring *in_cbr = &c->in_br;
+	int ret = 0;
+	void *data;
+
+	if (!cqe->res)
+		goto out;
+	if (cqe->flags & IORING_CQE_F_BUF_MORE)
+		return 0;
+
+	assert(cqe->res <= buf_size);
+	cd->out_bytes += cqe->res;
+
+	data = in_cbr->buf + bid * buf_size;
+	io_uring_buf_ring_add(in_cbr->br, data, buf_size, bid, br_mask, 0);
+	io_uring_buf_ring_advance(in_cbr->br, 1);
+	bid = (bid + 1) & (nr_bufs - 1);
+	ret = 1;
+out:
+	if (pending_shutdown(c))
+		close_cd(c, cd);
+
+	return ret;
+}
+
 /*
  * Handling a send with an outgoing send ring. Get the buffers from the
  * receive side, and add them to the ingoing buffer ring again.
  */
-static int handle_send_ring(struct conn *c, struct conn_dir *cd,
-			    int bid, int bytes)
+static int handle_send_ring(struct conn *c, struct conn_dir *cd, int bid,
+			    int bytes)
 {
 	struct conn_buf_ring *in_cbr = &c->in_br;
 	struct conn_buf_ring *out_cbr = &c->out_br;
@@ -1605,7 +1667,9 @@ static int __handle_send(struct io_uring *ring, struct conn *c,
 
 		vlog("send: got %d, %lu\n", cqe->res, cd->out_bytes);
 
-		if (send_ring)
+		if (buf_ring_inc)
+			nr_packets = handle_send_inc(c, cd, bid, cqe);
+		else if (send_ring)
 			nr_packets = handle_send_ring(c, cd, bid, cqe->res);
 		else
 			nr_packets = handle_send_buf(c, cd, bid, cqe->res);
@@ -2305,7 +2369,7 @@ int main(int argc, char *argv[])
 
 	pthread_mutex_init(&thread_lock, NULL);
 
-	optstring = "m:d:S:s:b:f:H:r:p:n:B:N:T:w:t:M:R:u:c:C:q:a:x:z:6Vh?";
+	optstring = "m:d:S:s:b:f:H:r:p:n:B:N:T:w:t:M:R:u:c:C:q:a:x:z:i:6Vh?";
 	while ((opt = getopt(argc, argv, optstring)) != -1) {
 		switch (opt) {
 		case 'm':
@@ -2377,6 +2441,9 @@ int main(int argc, char *argv[])
 		case 'q':
 			ring_size = atoi(optarg);
 			break;
+		case 'i':
+			buf_ring_inc = !!atoi(optarg);
+			break;
 		case 'a':
 			use_huge = !!atoi(optarg);
 			break;

data/vendor/liburing/examples/reg-wait.c

@@ -0,0 +1,159 @@
+/* SPDX-License-Identifier: MIT */
+/*
+ * Sample program that shows how to use registered waits.
+ *
+ * (C) 2024 Jens Axboe <axboe@kernel.dk>
+ */
+#include <stdint.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <unistd.h>
+#include <assert.h>
+#include <sys/time.h>
+#include <liburing.h>
+
+static unsigned long long mtime_since(const struct timeval *s,
+				      const struct timeval *e)
+{
+	long long sec, usec;
+
+	sec = e->tv_sec - s->tv_sec;
+	usec = (e->tv_usec - s->tv_usec);
+	if (sec > 0 && usec < 0) {
+		sec--;
+		usec += 1000000;
+	}
+
+	sec *= 1000;
+	usec /= 1000;
+	return sec + usec;
+}
+
+static unsigned long long mtime_since_now(struct timeval *tv)
+{
+	struct timeval end;
+
+	gettimeofday(&end, NULL);
+	return mtime_since(tv, &end);
+}
+
+int main(int argc, char *argv[])
+{
+	struct io_uring_reg_wait *reg;
+	struct io_uring_sqe *sqe;
+	struct io_uring_cqe *cqe[2];
+	struct io_uring ring;
+	char b1[8], b2[8];
+	unsigned long msec;
+	struct timeval tv;
+	int ret, fds[2];
+
+	if (argc > 1) {
+		fprintf(stdout, "%s: takes no arguments\n", argv[0]);
+		return 0;
+	}
+
+	if (pipe(fds) < 0) {
+		perror("pipe");
+		return 1;
+	}
+
+	ret = io_uring_queue_init(8, &ring, 0);
+	if (ret) {
+		fprintf(stderr, "Queue init: %d\n", ret);
+		return 1;
+	}
+
+	/*
+	 * Setup wait region. We'll use 32 here, but 64 is probably a more
+	 * logical value, as it'll pin a page regardless of size. 64 is the
+	 * max value on a 4k page size architecture.
+	 */
+	reg = io_uring_setup_reg_wait(&ring, 32, &ret);
+	if (!reg) {
+		if (ret == -EINVAL) {
+			fprintf(stderr, "Kernel doesn't support registered waits\n");
+			return 1;
+		}
+		fprintf(stderr, "Registered wait: %d\n", ret);
+		return 1;
+	}
+
+	/*
+	 * Setup two distinct wait regions. Index 0 will be a 1 second wait,
+	 * and region 2 is a short wait using min_wait_usec as well. Neither
+	 * of these use a signal mask, but sigmask/sigmask_sz can be set as
+	 * well for that.
+	 */
+	reg[0].ts.tv_sec = 1;
+	reg[0].ts.tv_nsec = 0;
+	reg[0].flags = IORING_REG_WAIT_TS;
+
+	reg[1].ts.tv_sec = 0;
+	reg[1].ts.tv_nsec = 100000000LL;
+	reg[1].min_wait_usec = 10000;
+	reg[1].flags = IORING_REG_WAIT_TS;
+
+	/*
+	 * No pending completions. Wait with region 0, which should time
+	 * out after 1 second.
+	 */
+	gettimeofday(&tv, NULL);
+	ret = io_uring_submit_and_wait_reg(&ring, cqe, 1, 0);
+	if (ret == -EINVAL) {
+		fprintf(stderr, "Kernel doesn't support registered waits\n");
+		return 1;
+	} else if (ret != -ETIME) {
+		fprintf(stderr, "Wait should've timed out... %d\n", ret);
+		return 1;
+	}
+	msec = mtime_since_now(&tv);
+	if (msec < 900 || msec > 1100) {
+		fprintf(stderr, "Wait took an unexpected amount of time: %lu\n",
+			msec);
+		return 1;
+	}
+
+	/*
+	 * Now prepare two pipe reads. We'll trigger one completion quickly,
+	 * but the other one will never happen. Use min_wait_usec timeout
+	 * to abort after 10 msec of time, where the overall timeout is
+	 * otherwise 100 msec. Since we're waiting on two events, the min
+	 * timeout ends up aborting us.
+	 */
+	sqe = io_uring_get_sqe(&ring);
+	io_uring_prep_read(sqe, fds[0], b1, sizeof(b1), 0);
+	sqe = io_uring_get_sqe(&ring);
+	io_uring_prep_read(sqe, fds[0], b2, sizeof(b2), 0);
+
+	/* trigger one read */
+	ret = write(fds[1], "Hello", 5);
+	if (ret < 0) {
+		perror("write");
+		return 1;
+	}
+
+	/*
+	 * This should will wait for 2 entries, where 1 is already available.
+	 * Since we're using min_wait_usec == 10 msec here with an overall
+	 * wait of 100 msec, we expect the wait to abort after 10 msec since
+	 * one or more events are available.
+	 */
+	gettimeofday(&tv, NULL);
+	ret = io_uring_submit_and_wait_reg(&ring, cqe, 2, 1);
+	msec = mtime_since_now(&tv);
+	if (ret != 2) {
+		fprintf(stderr, "Should have submitted 2: %d\n", ret);
+		return 1;
+	}
+	if (msec < 8 || msec > 12)
+		fprintf(stderr, "min_wait_usec should take ~10 msec: %lu\n", msec);
+
+	/*
+	 * Cleanup after ourselves
+	 */
+	io_uring_queue_exit(&ring);
+	io_uring_free_reg_wait(reg, 32);
+	return 0;
+}