uringmachine 0.1

data/vendor/liburing/test/helpers.c

@@ -0,0 +1,318 @@
+/* SPDX-License-Identifier: MIT */
+/*
+ * Description: Helpers for tests.
+ */
+#include <stdlib.h>
+#include <assert.h>
+#include <string.h>
+#include <stdio.h>
+#include <fcntl.h>
+#include <unistd.h>
+#include <stdarg.h>
+#include <sys/types.h>
+
+#include <arpa/inet.h>
+#include <netinet/ip.h>
+#include <netinet/tcp.h>
+
+#include "helpers.h"
+#include "liburing.h"
+
+/*
+ * Helper for allocating memory in tests.
+ */
+void *t_malloc(size_t size)
+{
+	void *ret;
+	ret = malloc(size);
+	assert(ret);
+	return ret;
+}
+
+/*
+ * Helper for binding socket to an ephemeral port.
+ * The port number to be bound is returned in @addr->sin_port.
+ */
+int t_bind_ephemeral_port(int fd, struct sockaddr_in *addr)
+{
+	socklen_t addrlen;
+	int ret;
+
+	addr->sin_port = 0;
+	if (bind(fd, (struct sockaddr *)addr, sizeof(*addr)))
+		return -errno;
+
+	addrlen = sizeof(*addr);
+	ret = getsockname(fd, (struct sockaddr *)addr, &addrlen);
+	assert(!ret);
+	assert(addr->sin_port != 0);
+	return 0;
+}
+
+/*
+ * Helper for allocating size bytes aligned on a boundary.
+ */
+void t_posix_memalign(void **memptr, size_t alignment, size_t size)
+{
+	int ret;
+	ret = posix_memalign(memptr, alignment, size);
+	assert(!ret);
+}
+
+/*
+ * Helper for allocating space for an array of nmemb elements
+ * with size bytes for each element.
+ */
+void *t_calloc(size_t nmemb, size_t size)
+{
+	void *ret;
+	ret = calloc(nmemb, size);
+	assert(ret);
+	return ret;
+}
+
+/*
+ * Helper for creating file and write @size byte buf with 0xaa value in the file.
+ */
+static void __t_create_file(const char *file, size_t size, char pattern)
+{
+	ssize_t ret;
+	char *buf;
+	int fd;
+
+	buf = t_malloc(size);
+	memset(buf, pattern, size);
+
+	fd = open(file, O_WRONLY | O_CREAT, 0644);
+	assert(fd >= 0);
+
+	ret = write(fd, buf, size);
+	fsync(fd);
+	close(fd);
+	free(buf);
+	assert(ret == size);
+}
+
+void t_create_file(const char *file, size_t size)
+{
+	__t_create_file(file, size, 0xaa);
+}
+
+void t_create_file_pattern(const char *file, size_t size, char pattern)
+{
+	__t_create_file(file, size, pattern);
+}
+
+/*
+ * Helper for creating @buf_num number of iovec
+ * with @buf_size bytes buffer of each iovec.
+ */
+struct iovec *t_create_buffers(size_t buf_num, size_t buf_size)
+{
+	struct iovec *vecs;
+	int i;
+
+	vecs = t_malloc(buf_num * sizeof(struct iovec));
+	for (i = 0; i < buf_num; i++) {
+		t_posix_memalign(&vecs[i].iov_base, buf_size, buf_size);
+		vecs[i].iov_len = buf_size;
+	}
+	return vecs;
+}
+
+/*
+ * Helper for setting up an io_uring instance, skipping if the given user isn't
+ * allowed to.
+ */
+enum t_setup_ret t_create_ring_params(int depth, struct io_uring *ring,
+				      struct io_uring_params *p)
+{
+	int ret;
+
+	ret = io_uring_queue_init_params(depth, ring, p);
+	if (!ret)
+		return T_SETUP_OK;
+	if ((p->flags & IORING_SETUP_SQPOLL) && ret == -EPERM && geteuid()) {
+		fprintf(stdout, "SQPOLL skipped for regular user\n");
+		return T_SETUP_SKIP;
+	}
+
+	if (ret != -EINVAL)
+		fprintf(stderr, "queue_init: %s\n", strerror(-ret));
+	return ret;
+}
+
+enum t_setup_ret t_create_ring(int depth, struct io_uring *ring,
+			       unsigned int flags)
+{
+	struct io_uring_params p = { };
+
+	p.flags = flags;
+	return t_create_ring_params(depth, ring, &p);
+}
+
+enum t_setup_ret t_register_buffers(struct io_uring *ring,
+				    const struct iovec *iovecs,
+				    unsigned nr_iovecs)
+{
+	int ret;
+
+	ret = io_uring_register_buffers(ring, iovecs, nr_iovecs);
+	if (!ret)
+		return T_SETUP_OK;
+
+	if ((ret == -EPERM || ret == -ENOMEM) && geteuid()) {
+		fprintf(stdout, "too large non-root buffer registration, skip\n");
+		return T_SETUP_SKIP;
+	}
+
+	fprintf(stderr, "buffer register failed: %s\n", strerror(-ret));
+	return ret;
+}
+
+int t_create_socket_pair(int fd[2], bool stream)
+{
+	int ret;
+	int type = stream ? SOCK_STREAM : SOCK_DGRAM;
+	int val;
+	struct sockaddr_in serv_addr;
+	struct sockaddr *paddr;
+	socklen_t paddrlen;
+
+	type |= SOCK_CLOEXEC;
+	fd[0] = socket(AF_INET, type, 0);
+	if (fd[0] < 0)
+		return errno;
+	fd[1] = socket(AF_INET, type, 0);
+	if (fd[1] < 0) {
+		ret = errno;
+		close(fd[0]);
+		return ret;
+	}
+
+	val = 1;
+	if (setsockopt(fd[0], SOL_SOCKET, SO_REUSEADDR, &val, sizeof(val)))
+		goto errno_cleanup;
+
+	memset(&serv_addr, 0, sizeof(serv_addr));
+	serv_addr.sin_family = AF_INET;
+	serv_addr.sin_port = 0;
+	inet_pton(AF_INET, "127.0.0.1", &serv_addr.sin_addr);
+
+	paddr = (struct sockaddr *)&serv_addr;
+	paddrlen = sizeof(serv_addr);
+
+	if (bind(fd[0], paddr, paddrlen)) {
+		fprintf(stderr, "bind failed\n");
+		goto errno_cleanup;
+	}
+
+	if (stream && listen(fd[0], 16)) {
+		fprintf(stderr, "listen failed\n");
+		goto errno_cleanup;
+	}
+
+	if (getsockname(fd[0], (struct sockaddr *)&serv_addr,
+			(socklen_t *)&paddrlen)) {
+		fprintf(stderr, "getsockname failed\n");
+		goto errno_cleanup;
+	}
+	inet_pton(AF_INET, "127.0.0.1", &serv_addr.sin_addr);
+
+	if (connect(fd[1], (struct sockaddr *)&serv_addr, paddrlen)) {
+		fprintf(stderr, "connect failed\n");
+		goto errno_cleanup;
+	}
+
+	if (!stream) {
+		/* connect the other udp side */
+		if (getsockname(fd[1], (struct sockaddr *)&serv_addr,
+				(socklen_t *)&paddrlen)) {
+			fprintf(stderr, "getsockname failed\n");
+			goto errno_cleanup;
+		}
+		inet_pton(AF_INET, "127.0.0.1", &serv_addr.sin_addr);
+
+		if (connect(fd[0], (struct sockaddr *)&serv_addr, paddrlen)) {
+			fprintf(stderr, "connect failed\n");
+			goto errno_cleanup;
+		}
+		return 0;
+	}
+
+	/* for stream case we must accept and cleanup the listen socket */
+
+	ret = accept(fd[0], NULL, NULL);
+	if (ret < 0)
+		goto errno_cleanup;
+
+	close(fd[0]);
+	fd[0] = ret;
+
+	return 0;
+
+errno_cleanup:
+	ret = errno;
+	close(fd[0]);
+	close(fd[1]);
+	return ret;
+}
+
+bool t_probe_defer_taskrun(void)
+{
+	struct io_uring ring;
+	int ret;
+
+	ret = io_uring_queue_init(1, &ring, IORING_SETUP_SINGLE_ISSUER |
+					    IORING_SETUP_DEFER_TASKRUN);
+	if (ret < 0)
+		return false;
+	io_uring_queue_exit(&ring);
+	return true;
+}
+
+/*
+ * Sync internal state with kernel ring state on the SQ side. Returns the
+ * number of pending items in the SQ ring, for the shared ring.
+ */
+unsigned __io_uring_flush_sq(struct io_uring *ring)
+{
+	struct io_uring_sq *sq = &ring->sq;
+	unsigned tail = sq->sqe_tail;
+
+	if (sq->sqe_head != tail) {
+		sq->sqe_head = tail;
+		/*
+		 * Ensure kernel sees the SQE updates before the tail update.
+		 */
+		if (!(ring->flags & IORING_SETUP_SQPOLL))
+			*sq->ktail = tail;
+		else
+			io_uring_smp_store_release(sq->ktail, tail);
+	}
+	/*
+	* This load needs to be atomic, since sq->khead is written concurrently
+	* by the kernel, but it doesn't need to be load_acquire, since the
+	* kernel doesn't store to the submission queue; it advances khead just
+	* to indicate that it's finished reading the submission queue entries
+	* so they're available for us to write to.
+	*/
+	return tail - IO_URING_READ_ONCE(*sq->khead);
+}
+
+/*
+ * Implementation of error(3), prints an error message and exits.
+ */
+void t_error(int status, int errnum, const char *format, ...)
+{
+	va_list args;
+    	va_start(args, format);
+
+	vfprintf(stderr, format, args);
+    	if (errnum)
+        	fprintf(stderr, ": %s", strerror(errnum));
+
+	fprintf(stderr, "\n");
+	va_end(args);
+    	exit(status);
+}

data/vendor/liburing/test/helpers.h

@@ -0,0 +1,108 @@
+/* SPDX-License-Identifier: MIT */
+/*
+ * Description: Helpers for tests.
+ */
+#ifndef LIBURING_HELPERS_H
+#define LIBURING_HELPERS_H
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#include "liburing.h"
+#include "../src/setup.h"
+#include <arpa/inet.h>
+
+enum t_setup_ret {
+	T_SETUP_OK	= 0,
+	T_SETUP_SKIP,
+};
+
+enum t_test_result {
+	T_EXIT_PASS   = 0,
+	T_EXIT_FAIL   = 1,
+	T_EXIT_SKIP   = 77,
+};
+
+/*
+ * Helper for binding socket to an ephemeral port.
+ * The port number to be bound is returned in @addr->sin_port.
+ */
+int t_bind_ephemeral_port(int fd, struct sockaddr_in *addr);
+
+
+/*
+ * Helper for allocating memory in tests.
+ */
+void *t_malloc(size_t size);
+
+
+/*
+ * Helper for allocating size bytes aligned on a boundary.
+ */
+void t_posix_memalign(void **memptr, size_t alignment, size_t size);
+
+
+/*
+ * Helper for allocating space for an array of nmemb elements
+ * with size bytes for each element.
+ */
+void *t_calloc(size_t nmemb, size_t size);
+
+
+/*
+ * Helper for creating file and write @size byte buf with 0xaa value in the file.
+ */
+void t_create_file(const char *file, size_t size);
+
+/*
+ * Helper for creating file and write @size byte buf with @pattern value in
+ * the file.
+ */
+void t_create_file_pattern(const char *file, size_t size, char pattern);
+
+/*
+ * Helper for creating @buf_num number of iovec
+ * with @buf_size bytes buffer of each iovec.
+ */
+struct iovec *t_create_buffers(size_t buf_num, size_t buf_size);
+
+/*
+ * Helper for creating connected socket pairs
+ */
+int t_create_socket_pair(int fd[2], bool stream);
+
+/*
+ * Helper for setting up a ring and checking for user privs
+ */
+enum t_setup_ret t_create_ring_params(int depth, struct io_uring *ring,
+				      struct io_uring_params *p);
+enum t_setup_ret t_create_ring(int depth, struct io_uring *ring,
+			       unsigned int flags);
+
+enum t_setup_ret t_register_buffers(struct io_uring *ring,
+				    const struct iovec *iovecs,
+				    unsigned nr_iovecs);
+
+bool t_probe_defer_taskrun(void);
+
+unsigned __io_uring_flush_sq(struct io_uring *ring);
+
+static inline int t_io_uring_init_sqarray(unsigned entries, struct io_uring *ring,
+					struct io_uring_params *p)
+{
+	int ret;
+
+	ret = __io_uring_queue_init_params(entries, ring, p, NULL, 0);
+	return ret >= 0 ? 0 : ret;
+}
+
+#define ARRAY_SIZE(x) (sizeof(x) / sizeof((x)[0]))
+
+void t_error(int status, int errnum, const char *format, ...);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif

data/vendor/liburing/test/ignore-single-mmap.c

@@ -0,0 +1,48 @@
+/* SPDX-License-Identifier: MIT */
+/*
+ * 6.10-rc merge window had a bug where the rewritten mmap support caused
+ * rings allocated with > 1 page, but asking for smaller mappings, would
+ * cause -EFAULT to be returned rather than a successful map. This hit
+ * applications either using an ancient liburing with IORING_FEAT_SINGLE_MMAP
+ * support, or application just ignoring that feature flag and still doing
+ * 3 mmap operations to map the ring.
+ */
+#include <stdio.h>
+#include <stdlib.h>
+#include <unistd.h>
+
+#include "../src/syscall.h"
+#include "liburing.h"
+#include "helpers.h"
+
+#define ENTRIES	128
+
+int main(int argc, char *argv[])
+{
+	struct io_uring_params p = { };
+	void *ptr;
+	int fd;
+
+	if (argc > 1)
+		return T_EXIT_SKIP;
+
+	fd = __sys_io_uring_setup(ENTRIES, &p);
+	if (fd < 0)
+		return T_EXIT_SKIP;
+
+	if (!(p.features & IORING_FEAT_SINGLE_MMAP)) {
+		close(fd);
+		return T_EXIT_SKIP;
+	}
+
+	ptr = __sys_mmap(0, ENTRIES * sizeof(unsigned), PROT_READ | PROT_WRITE,
+				MAP_SHARED | MAP_POPULATE, fd,
+				IORING_OFF_SQ_RING);
+	if (!IS_ERR(ptr)) {
+		close(fd);
+		return T_EXIT_PASS;
+	}
+
+	fprintf(stderr, "ring sqe array mmap: %d\n", PTR_ERR(ptr));
+	return T_EXIT_FAIL;
+}

data/vendor/liburing/test/init-mem.c

@@ -0,0 +1,164 @@
+/* SPDX-License-Identifier: MIT */
+/*
+ * Description: Check that io_uring_queue_init_mem() doesn't underestimate
+ *		the memory required for various size rings.
+ */
+#include <stdio.h>
+#include <unistd.h>
+#include <errno.h>
+#include <sys/mman.h>
+#include <linux/mman.h>
+#include <stdlib.h>
+#include <string.h>
+#include <netinet/udp.h>
+#include <arpa/inet.h>
+#include <net/if.h>
+#include <error.h>
+
+#include "liburing.h"
+#include "helpers.h"
+
+#define PRE_RED	0x5aa55aa55aa55aa5ULL
+#define POST_RED 0xa55aa55aa55aa55aULL
+
+struct ctx {
+	struct io_uring		ring;
+	void			*ring_mem;
+	void			*mem;
+	unsigned long long	*pre;
+	unsigned long long	*post;
+};
+
+struct q_entries {
+	unsigned int sqes;
+	unsigned int cqes;
+};
+
+static int setup_ctx(struct ctx *ctx, struct q_entries *q)
+{
+	struct io_uring_params p = { };
+	int ret;
+
+	if (posix_memalign(&ctx->mem, 4096, 2*1024*1024))
+		return T_EXIT_FAIL;
+
+	ctx->pre = ctx->mem + 4096 - sizeof(unsigned long long);
+	*ctx->pre = PRE_RED;
+
+	ctx->ring_mem = ctx->mem + 4096;
+	p.flags |= IORING_SETUP_CQSIZE | IORING_SETUP_NO_SQARRAY;
+	p.sq_entries = q->sqes;
+	p.cq_entries = q->cqes;
+
+	ret = io_uring_queue_init_mem(q->sqes, &ctx->ring, &p,
+					ctx->ring_mem, 2*1024*1024);
+
+	if (ret < 0) {
+		if (ret == -EINVAL)
+			return T_EXIT_SKIP;
+		fprintf(stderr, "queue init: %d\n", ret);
+		return T_EXIT_FAIL;
+	}
+
+	ctx->post = ctx->ring_mem + ret;
+	*ctx->post = POST_RED;
+	return 0;
+}
+
+static void clean_ctx(struct ctx *ctx)
+{
+	io_uring_queue_exit(&ctx->ring);
+}
+
+static int check_red(struct ctx *ctx, unsigned long i)
+{
+	int fail = 0;
+
+	if (*ctx->pre != PRE_RED) {
+		printf("pre redzone=%llx at i=%lu\n", *ctx->pre, i);
+		fail = 1;
+	}
+	if (*ctx->post != POST_RED) {
+		printf("post redzone=%llx at i=%lu\n", *ctx->post, i);
+		fail = 1;
+	}
+	return fail;
+}
+
+static int test(struct q_entries *q)
+{
+	struct io_uring_sqe *sqe;
+	struct io_uring_cqe *cqe;
+	struct ctx ctx = { };
+	unsigned long i, ud;
+	int j, ret, batch;
+
+	ret = setup_ctx(&ctx, q);
+	if (ret == T_EXIT_SKIP)
+		return T_EXIT_SKIP;
+	else if (ret != T_EXIT_PASS)
+		return ret;
+
+	batch = 64;
+	if (batch > q->sqes)
+		batch = q->sqes;
+
+	i = ud = 0;
+	while (i < q->cqes * 2) {
+		if (check_red(&ctx, i))
+			return T_EXIT_FAIL;
+		for (j = 0; j < batch; j++) {
+			sqe = io_uring_get_sqe(&ctx.ring);
+			io_uring_prep_nop(sqe);
+			sqe->user_data = j + (unsigned long) i;
+		}
+		io_uring_submit(&ctx.ring);
+		for (j = 0; j < batch; j++) {
+			ret = io_uring_wait_cqe(&ctx.ring, &cqe);
+			if (ret)
+				goto err;
+			if (cqe->user_data != ud) {
+				fprintf(stderr, "ud=%lu, wanted %lu\n", (unsigned long) cqe->user_data, ud);
+				goto err;
+			}
+			ud++;
+			io_uring_cqe_seen(&ctx.ring, cqe);
+		}
+		i += batch;
+	}
+
+	clean_ctx(&ctx);
+	return T_EXIT_PASS;
+err:
+	clean_ctx(&ctx);
+	return T_EXIT_FAIL;
+}
+
+int main(int argc, char *argv[])
+{
+	struct q_entries q_entries[] = {
+		{ 256, 16384 },
+		{ 32, 4096 },
+		{ 128, 8192 },
+		{ 4096, 32768 },
+		{ 1, 8 },
+		{ 2, 1024 },
+	};
+	int i, ret;
+
+	if (argc > 1)
+		return T_EXIT_SKIP;
+
+	for (i = 0; i < ARRAY_SIZE(q_entries); i++) {
+		ret = test(&q_entries[i]);
+		if (ret == T_EXIT_SKIP) {
+			return T_EXIT_SKIP;
+		} else if (ret != T_EXIT_PASS) {
+			fprintf(stderr, "Failed at %d/%d\n", q_entries[i].sqes,
+							q_entries[i].cqes);
+			return T_EXIT_FAIL;
+		}
+	}
+
+	return T_EXIT_PASS;
+}