uringmachine 0.24.0 → 0.26.0

data/vendor/liburing/test/sendzc-bug.c

@@ -0,0 +1,146 @@
+/* SPDX-License-Identifier: MIT */
+/*
+ * Description: test that importing a buffer node ties the lifetime to the
+ *		notification io_kiocb, not the normal request.
+ *
+ * Based on a test case from Google Big Sleep.
+ *
+ */
+#include <arpa/inet.h>
+#include <err.h>
+#include <fcntl.h>
+#include <netinet/in.h>
+#include <poll.h>
+#include <pthread.h>
+#include <stdint.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <sys/mman.h>
+#include <sys/socket.h>
+#include <sys/stat.h>
+#include <sys/syscall.h>
+#include <sys/types.h>
+#include <sys/wait.h>
+#include <unistd.h>
+
+#include "liburing.h"
+#include "helpers.h"
+
+#define BUF_SIZE (1024 * 1024)
+static uint8_t read_buffer[BUF_SIZE] = { 0 };
+
+/* Creates a pair of connected TCP sockets. */
+static void tcp_socketpair(int sv[2])
+{
+	struct sockaddr_in addr_in = {
+		.sin_family = AF_INET,
+		.sin_addr.s_addr = htonl(INADDR_LOOPBACK),
+		.sin_port = 0,
+	};
+	struct sockaddr *addr = (struct sockaddr *) &addr_in;
+	socklen_t addr_len = sizeof(addr_in);
+	int listener_fd;
+
+	listener_fd = socket(AF_INET, SOCK_STREAM, 0);
+	if (listener_fd < 0)
+		err(EXIT_FAILURE, "listener socket");
+
+	if (bind(listener_fd, addr, addr_len) < 0)
+		err(EXIT_FAILURE, "bind");
+
+	if (listen(listener_fd, 1) < 0)
+		err(EXIT_FAILURE, "listen");
+
+	if (getsockname(listener_fd, addr, &addr_len) < 0)
+		err(EXIT_FAILURE, "getsockname");
+
+	sv[0] = socket(AF_INET, SOCK_STREAM, 0);
+	if (sv[0] < 0)
+		err(EXIT_FAILURE, "socket(AF_INET, SOCK_STREAM, 0)");
+
+	if (connect(sv[0], addr, addr_len) < 0)
+		err(EXIT_FAILURE, "connect");
+
+	sv[1] = accept(listener_fd, NULL, NULL);
+	if (sv[1] == -1)
+		err(EXIT_FAILURE, "accept");
+
+	close(listener_fd);
+}
+
+int main(int argc, char *argv[])
+{
+	struct io_uring_sqe *sqe;
+	struct io_uring_cqe* cqe;
+	struct iovec send_iov;
+	struct io_uring ring;
+	struct iovec iov_reg;
+	struct msghdr msg = {
+		.msg_iov = &send_iov,
+		.msg_iovlen = 1,
+	};
+	void *send_buffer;
+	int sockets[2];
+	int val, ret;
+
+	if (argc > 1)
+		return T_EXIT_SKIP;
+
+	tcp_socketpair(sockets);
+
+	val = 1;
+	if (setsockopt(sockets[0], SOL_SOCKET, SO_ZEROCOPY, &val, sizeof(val)))
+		err(EXIT_FAILURE, "setsockopt(SO_ZEROCOPY)");
+
+	ret = io_uring_queue_init(1, &ring, 0);
+	if (ret < 0)
+		errx(EXIT_FAILURE, "io_uring_queue_init: %s", strerror(-ret));
+
+	/* mmap send_buffer so we can unmap it later. */
+	send_buffer = mmap(NULL, BUF_SIZE, PROT_READ | PROT_WRITE,
+			  MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
+	if (send_buffer == MAP_FAILED)
+		err(EXIT_FAILURE, "mapping iouring buffer");
+
+	iov_reg.iov_base = send_buffer;
+	iov_reg.iov_len = BUF_SIZE;
+	ret = io_uring_register_buffers(&ring, &iov_reg, 1);
+	if (ret < 0)
+		errx(EXIT_FAILURE, "io_uring_register_buffers: %s", strerror(-ret));
+
+	sqe = io_uring_get_sqe(&ring);
+
+	send_iov.iov_base = send_buffer;
+	send_iov.iov_len = BUF_SIZE;
+
+	io_uring_prep_sendmsg_zc(sqe, sockets[0], &msg, 0);
+	sqe->ioprio = IORING_RECVSEND_FIXED_BUF;
+	sqe->buf_index = 0;
+
+	memset(send_buffer, 'B', BUF_SIZE);
+
+	io_uring_submit(&ring);
+
+	ret = io_uring_wait_cqe(&ring, &cqe);
+	if (ret < 0)
+		errx(EXIT_FAILURE, "io_uring_wait_cqe: %s", strerror(-ret));
+
+	if (cqe->res < 0) {
+		if (cqe->res == -EINVAL)
+			return T_EXIT_SKIP;
+		errx(EXIT_FAILURE, "readv failed: %s", strerror(-cqe->res));
+	}
+
+	io_uring_cqe_seen(&ring, cqe);
+
+	io_uring_unregister_buffers(&ring);
+	munmap(send_buffer, BUF_SIZE);
+
+	ret = read(sockets[1], read_buffer, BUF_SIZE);
+	if (ret < 0)
+		err(EXIT_FAILURE, "read");
+
+	io_uring_queue_exit(&ring);
+	return T_EXIT_PASS;
+}

data/vendor/liburing/test/sqe-mixed-nop.c

@@ -9,9 +9,11 @@
 #include "helpers.h"
 #include "test.h"
 
+#define NENTRIES	8
+
 static int seq;
 
-static int test_single_nop(struct io_uring *ring, bool sqe128)
+static int test_single_nop(struct io_uring *ring, bool sqe128, int offset)
 {
 	struct io_uring_cqe *cqe;
 	struct io_uring_sqe *sqe;
@@ -54,15 +56,123 @@ static int test_single_nop(struct io_uring *ring, bool sqe128)
 	return T_EXIT_FAIL;
 }
 
-int main(int argc, char *argv[])
+static int check_sq(struct io_uring *ring, unsigned int ready,
+		    unsigned int left, const char *msg)
+{
+	unsigned int pspace, pready;
+
+	pready = io_uring_sq_ready(ring);
+	if (pready != ready) {
+		fprintf(stderr, "%s: %d ready, should be %d\n", msg, pready, ready);
+		return 1;
+	}
+	pspace = io_uring_sq_space_left(ring);
+	if (pspace != left) {
+		fprintf(stderr, "%s: %d entries, should be %d\n", msg, pspace, left);
+		return 1;
+	}
+
+	return 0;
+}
+
+static void submit_and_reap(struct io_uring *ring, int to_reap, const char *msg)
+{
+	struct io_uring_cqe *cqe;
+	unsigned long ud = 1000;
+	int i;
+
+	io_uring_submit(ring);
+
+	for (i = 0; i < to_reap; i++) {
+		io_uring_wait_cqe(ring, &cqe);
+		if (cqe->res < 0) {
+			fprintf(stderr, "cqe res %d\n", cqe->res);
+			io_uring_cqe_seen(ring, cqe);
+			continue;
+		}
+		if (cqe->user_data != ud)
+			fprintf(stderr, "%s: cqe=%d, %ld != %ld\n", msg, i, (long) cqe->user_data, ud);
+		ud++;
+		io_uring_cqe_seen(ring, cqe);
+	}
+}
+
+static int test_sq_space(struct io_uring *ring, unsigned int flags)
+{
+	struct io_uring_sqe *sqe;
+	int nop_adj;
+
+	/*
+	 * For SQ_REWIND, sqe_tail resets to 0 after each submit, so we always
+	 * start at position 0 and never hit wrap cases. For non-SQ_REWIND,
+	 * after 32 test_single_nop iterations, sqe_tail ends up at position 7
+	 * (modulo 8), so step5's sqe128 triggers the wrap case with a NOP.
+	 */
+	nop_adj = 0;
+	if (!(flags & IORING_SETUP_SQ_REWIND))
+		nop_adj = 1;
+
+	if (check_sq(ring, 0, NENTRIES, "initial"))
+		return 1;
+
+	sqe = io_uring_get_sqe(ring);
+	io_uring_prep_nop(sqe);
+	sqe->user_data = 1000;
+
+	if (check_sq(ring, 1, NENTRIES - 1, "step1"))
+		return 1;
+
+	sqe = io_uring_get_sqe128(ring);
+	io_uring_prep_nop128(sqe);
+	sqe->user_data = 1001;
+
+	if (check_sq(ring, 3, NENTRIES - 3, "step2"))
+		return 1;
+
+	sqe = io_uring_get_sqe(ring);
+	io_uring_prep_nop(sqe);
+	sqe->user_data = 1002;
+
+	if (check_sq(ring, 4, NENTRIES - 4, "step3"))
+		return 1;
+
+	submit_and_reap(ring, 3, "round1");
+
+	if (check_sq(ring, 0, NENTRIES, "step4"))
+		return 1;
+
+	sqe = io_uring_get_sqe128(ring);
+	io_uring_prep_nop128(sqe);
+	sqe->user_data = 1000;
+
+	/* SQ_REWIND: no wrap at position 0, just 2 slots. Non-rewind: wrap at 7, 3 slots */
+	if (check_sq(ring, 2 + nop_adj, NENTRIES - 2 - nop_adj, "step5"))
+		return 1;
+
+	sqe = io_uring_get_sqe(ring);
+	io_uring_prep_nop(sqe);
+	sqe->user_data = 1001;
+
+	if (check_sq(ring, 3 + nop_adj, NENTRIES - 3 - nop_adj, "step6"))
+		return 1;
+
+	sqe = io_uring_get_sqe128(ring);
+	io_uring_prep_nop128(sqe);
+	sqe->user_data = 1002;
+
+	if (check_sq(ring, 5 + nop_adj, NENTRIES - 5 - nop_adj, "step7"))
+		return 1;
+
+	submit_and_reap(ring, 3, "round2");
+	return 0;
+}
+
+static int test_flags(unsigned int flags)
 {
 	struct io_uring ring;
 	int ret, i;
 
-	if (argc > 1)
-		return T_EXIT_SKIP;
-
-	ret = io_uring_queue_init(8, &ring, IORING_SETUP_SQE_MIXED);
+	ret = io_uring_queue_init(NENTRIES, &ring, IORING_SETUP_SQE_MIXED | flags);
 	if (ret) {
 		if (ret == -EINVAL)
 			return T_EXIT_SKIP;
@@ -72,11 +182,45 @@ int main(int argc, char *argv[])
 
 	/* alternate big and little sqe's */
 	for (i = 0; i < 32; i++) {
-		ret = test_single_nop(&ring, i & 1);
+		ret = test_single_nop(&ring, i & 1, i);
 		if (ret != T_EXIT_PASS)
 			break;
 	}
 
+	if (ret) {
+		fprintf(stderr, "Single nop test failed\n");
+		io_uring_queue_exit(&ring);
+		return ret;
+	}
+
+	ret = test_sq_space(&ring, flags);
 	io_uring_queue_exit(&ring);
 	return ret;
+
+}
+
+int main(int argc, char *argv[])
+{
+	int ret;
+
+	if (argc > 1)
+		return T_EXIT_SKIP;
+
+	ret = test_flags(0);
+	if (ret == T_EXIT_SKIP) {
+		return T_EXIT_SKIP;
+	} else if (ret) {
+		fprintf(stderr, "test flags 0 failed\n");
+		return T_EXIT_FAIL;
+	}
+
+	ret = test_flags(IORING_SETUP_SQ_REWIND);
+	if (ret == T_EXIT_SKIP) {
+		return T_EXIT_SKIP;
+	} else if (ret) {
+		fprintf(stderr, "test flags REWIND failed\n");
+		return T_EXIT_FAIL;
+	}
+
+	return ret;
 }

data/vendor/liburing/test/test.h

@@ -20,6 +20,8 @@ static io_uring_test_config io_uring_test_configs[] = {
 	{ IORING_SETUP_SQE128, 				"large SQE"},
 	{ IORING_SETUP_CQE32, 				"large CQE"},
 	{ IORING_SETUP_SQE128 | IORING_SETUP_CQE32, 	"large SQE/CQE" },
+	{ IORING_SETUP_SQ_REWIND,			"rewind SQ"},
+	{ IORING_SETUP_SQ_REWIND | IORING_SETUP_SQE128,	"large rewind SQ"},
 };
 
 #define FOR_ALL_TEST_CONFIGS							\

data/vendor/liburing/test/timestamp-bug.c

@@ -0,0 +1,135 @@
+/* SPDX-License-Identifier: MIT */
+/*
+ * Description: verify that timestamp retrieval that needs retrying with
+ *		a current list of SKBs works.
+ *
+ * Based on a test case from Google Big Sleep.
+ *
+ */
+#include <arpa/inet.h>
+#include <err.h>
+#include <fcntl.h>
+#include <sys/time.h>
+#include <linux/errqueue.h>
+#include <linux/net_tstamp.h>
+#include <netinet/in.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <sys/socket.h>
+#include <sys/uio.h>
+#include <unistd.h>
+
+#include "liburing.h"
+#include "helpers.h"
+
+/* Compatibility with slightly older kernel headers */
+#ifndef SOCKET_URING_OP_TX_TIMESTAMP
+#define SOCKET_URING_OP_TX_TIMESTAMP 4
+#endif
+
+/*
+ * Create a socket whose error queue contains both timestamp information and
+ * actual errors.
+ */
+static int create_sock_with_timestamps_and_errors(void)
+{
+	struct sockaddr_in recv_addr = { };
+	struct sockaddr_in closed = { };
+	int recv_sock, sock, val;
+	char data = 'A';
+
+	sock = socket(AF_INET, SOCK_DGRAM, 0);
+	if (sock < 0)
+		err(1, "socket");
+
+	/* Enable TX timestamps */
+	val = SOF_TIMESTAMPING_SOFTWARE | SOF_TIMESTAMPING_TX_SOFTWARE |
+		SOF_TIMESTAMPING_OPT_ID | SOF_TIMESTAMPING_OPT_TSONLY;
+	if (setsockopt(sock, SOL_SOCKET, SO_TIMESTAMPING, &val, sizeof(val)) < 0)
+		err(1, "setsockopt(SO_TIMESTAMPING)");
+
+	/* Enable IP_RECVERR */
+	val = 1;
+	if (setsockopt(sock, IPPROTO_IP, IP_RECVERR, &val, sizeof(val)) < 0)
+		err(1, "setsockopt(IP_RECVERR)");
+
+	recv_sock = socket(AF_INET, SOCK_DGRAM, 0);
+	if (recv_sock < 0)
+		err(1, "socket");
+
+	recv_addr.sin_family = AF_INET;
+	recv_addr.sin_addr.s_addr = inet_addr("127.0.0.1");
+	if (bind(recv_sock, (struct sockaddr*)&recv_addr, sizeof(recv_addr)) < 0)
+		err(1, "bind receiver");
+
+	socklen_t recv_len = sizeof(recv_addr);
+	if (getsockname(recv_sock, (struct sockaddr*)&recv_addr, &recv_len) < 0)
+		err(1, "getsockname");
+
+	/* Send packets to generate TX timestamps. */
+	for (int i = 0; i < 2; i++) {
+		if (sendto(sock, &data, sizeof(data), 0, (struct sockaddr*)&recv_addr, sizeof(recv_addr)) < 0)
+			err(1, "sendto (%d)", i);
+	}
+
+	closed.sin_family = AF_INET;
+	closed.sin_addr.s_addr = inet_addr("127.0.0.1");
+	closed.sin_port = htons(9); /* assuming no one is listening on port 9 */
+	if (sendto(sock, &data, sizeof(data), 0, (struct sockaddr*) &closed, sizeof(closed)) < 0)
+		warn("sendto");
+
+	return sock;
+}
+
+int main(int argc, char *argv[])
+{
+	struct io_uring_sqe *sqe;
+	struct io_uring_cqe *cqe;
+	struct io_uring ring;
+	struct io_uring_params params = {
+		.flags = IORING_SETUP_CQE32, /* required for timestamps */
+	};
+	char buf[1024], ctrl_recv[1024];
+	struct iovec iov_recv = { .iov_base = buf, .iov_len = sizeof(buf) };
+	struct msghdr msg_recv = {
+		.msg_iov = &iov_recv,
+		.msg_iovlen = 1,
+		.msg_control = ctrl_recv,
+		.msg_controllen = sizeof(ctrl_recv),
+	};
+	int sock, ret;
+
+	if (argc > 1)
+		return T_EXIT_SKIP;
+
+	sock = create_sock_with_timestamps_and_errors();
+
+	ret = io_uring_queue_init_params(1, &ring, &params);
+	if (ret < 0) {
+		if (ret == -EINVAL)
+			return T_EXIT_SKIP;
+		err(1, "io_uring_queue_init_params: %d", ret);
+	}
+
+	sqe = io_uring_get_sqe(&ring);
+	*sqe = (struct io_uring_sqe) {
+		.opcode = IORING_OP_URING_CMD,
+		.fd = sock,
+		.cmd_op = SOCKET_URING_OP_TX_TIMESTAMP,
+	};
+
+	io_uring_submit(&ring);
+
+	ret = io_uring_wait_cqe(&ring, &cqe);
+	if (ret < 0)
+		errx(1, "io_uring_wait_cqe: %d", ret);
+	ret = cqe->res;
+	io_uring_cqe_seen(&ring, cqe);
+	/* kernel doesn't support retrieving timestamps */
+	if (ret == -EOPNOTSUPP)
+		return T_EXIT_SKIP;
+
+	recvmsg(sock, &msg_recv, MSG_ERRQUEUE | MSG_DONTWAIT);
+	return T_EXIT_PASS;
+}

data/vendor/liburing/test/timestamp.c

@@ -1,3 +1,8 @@
+/* SPDX-License-Identifier: MIT */
+/*
+ * Description: test timestamp retrieving
+ *
+ */
 #include <time.h>
 #include <arpa/inet.h>
 #include <inttypes.h>

data/vendor/liburing/test/vec-regbuf.c

@@ -520,6 +520,135 @@ static void test_basic(struct buf_desc *bd)
 	}
 }
 
+static int test_readv_fixed(struct buf_desc *bd, struct iovec *vecs, int nr_vec)
+{
+	unsigned buf_idx = bd->buf_idx;
+	struct io_uring *ring = &bd->ring;
+	struct io_uring_sqe *sqe;
+	struct io_uring_cqe *cqe;
+	size_t total_len, i, offset;
+	int ret, fd;
+	char *expected;
+	char filename[] = ".readv_fixed.XXXXXX";
+
+	total_len = t_iovec_data_length(vecs, nr_vec);
+
+	expected = malloc(total_len);
+	if (!expected) {
+		fprintf(stderr, "malloc failed\n");
+		return 1;
+	}
+
+	/* Fill expected buffer with pattern */
+	for (i = 0; i < total_len; i++)
+		expected[i] = i & 0xff;
+
+	fd = mkstemp(filename);
+	if (fd < 0) {
+		perror("mkstemp");
+		free(expected);
+		return 1;
+	}
+	unlink(filename);
+
+	/* Write expected data to file */
+	ret = write(fd, expected, total_len);
+	if (ret != total_len) {
+		fprintf(stderr, "file write failed: %d\n", ret);
+		close(fd);
+		free(expected);
+		return 1;
+	}
+
+	/* Clear registered buffer before reading into it */
+	memset(bd->buf_wr, 0, bd->size);
+
+	/* Read using READV_FIXED - vecs already point to bd->buf_wr (registered) */
+	sqe = io_uring_get_sqe(ring);
+	io_uring_prep_readv_fixed(sqe, fd, vecs, nr_vec, 0, 0, buf_idx);
+
+	ret = io_uring_submit(ring);
+	if (ret != 1) {
+		fprintf(stderr, "submit failed %i\n", ret);
+		close(fd);
+		free(expected);
+		return 1;
+	}
+
+	ret = io_uring_wait_cqe(ring, &cqe);
+	if (ret) {
+		fprintf(stderr, "wait_cqe=%d\n", ret);
+		close(fd);
+		free(expected);
+		return 1;
+	}
+
+	ret = cqe->res;
+	io_uring_cqe_seen(ring, cqe);
+
+	close(fd);
+
+	if (ret != total_len) {
+		fprintf(stderr, "readv_fixed: got %d, expected %zu\n", ret, total_len);
+		free(expected);
+		return 1;
+	}
+
+	/* Verify data across all iovec segments */
+	offset = 0;
+	for (i = 0; i < nr_vec; i++) {
+		if (memcmp(vecs[i].iov_base, expected + offset, vecs[i].iov_len) != 0) {
+			fprintf(stderr, "readv_fixed: data mismatch at segment %zu\n", i);
+			free(expected);
+			return 1;
+		}
+		offset += vecs[i].iov_len;
+	}
+
+	free(expected);
+	return 0;
+}
+
+static int test_readv(struct buf_desc *bd)
+{
+	void *p = bd->buf_wr;
+	int ret;
+	struct iovec iov_page = { .iov_base = p, .iov_len = page_sz };
+	struct iovec iov_multi[] = {
+		{ .iov_base = p, .iov_len = page_sz },
+		{ .iov_base = p + page_sz, .iov_len = page_sz },
+		{ .iov_base = p + page_sz * 2, .iov_len = page_sz },
+	};
+	struct iovec iov_unalign = { .iov_base = p + 1, .iov_len = page_sz - 1 };
+
+	reinit_ring(bd);
+
+	/* Single page read */
+	ret = test_readv_fixed(bd, &iov_page, 1);
+	if (ret) {
+		fprintf(stderr, "readv_fixed: single page failed\n");
+		return ret;
+	}
+
+	/* Multi-segment read */
+	reinit_ring(bd);
+	ret = test_readv_fixed(bd, iov_multi, 3);
+	if (ret) {
+		fprintf(stderr, "readv_fixed: multi-segment failed\n");
+		return ret;
+	}
+
+	/* Unaligned read */
+	reinit_ring(bd);
+	ret = test_readv_fixed(bd, &iov_unalign, 1);
+	if (ret) {
+		fprintf(stderr, "readv_fixed: unaligned failed\n");
+		return ret;
+	}
+
+	return 0;
+}
+
 static void test_fail(struct buf_desc *bd)
 {
 	int ret, cqe_ret;
@@ -576,7 +705,7 @@ static void test_fail(struct buf_desc *bd)
 int main(int argc, char *argv[])
 {
 	struct buf_desc bd = {};
-	int i = 0;
+	int ret, i;
 
 	if (argc > 1)
 		return T_EXIT_SKIP;
@@ -601,6 +730,12 @@ int main(int argc, char *argv[])
 		test_fail(&bd);
 	}
 
+	ret = test_readv(&bd);
+	if (ret) {
+		fprintf(stderr, "test_readv failed\n");
+		return T_EXIT_FAIL;
+	}
+
 	free(bd.buf_rd);
 	io_uring_queue_exit(&bd.ring);
 	return 0;