uringmachine 0.10 → 0.11

data/vendor/liburing/test/fixed-seg.c

@@ -0,0 +1,187 @@
+/* SPDX-License-Identifier: MIT */
+/*
+ * Description: test various offset of fixed buffer read
+ *
+ */
+#include <stdio.h>
+#include <unistd.h>
+#include <fcntl.h>
+#include <stdlib.h>
+#include <string.h>
+#include <sys/time.h>
+#include <sys/ioctl.h>
+#include <sys/stat.h>
+#include "liburing.h"
+#include "helpers.h"
+
+static struct iovec rvec, wvec;
+
+static int read_it(struct io_uring *ring, int fd, int len, int off)
+{
+	struct io_uring_sqe *sqe;
+	struct io_uring_cqe *cqe;
+	int ret;
+
+	sqe = io_uring_get_sqe(ring);
+	io_uring_prep_read_fixed(sqe, fd, rvec.iov_base + off, len, 0, 0);
+	sqe->user_data = 1;
+
+	io_uring_submit(ring);
+
+	ret = io_uring_wait_cqe(ring, &cqe);
+	if (ret) {
+		fprintf(stderr, "wait %d\n", ret);
+		return 1;
+	}
+	if (cqe->res < 0) {
+		fprintf(stderr, "cqe res %s\n", strerror(-cqe->res));
+		return 1;
+	}
+	if (cqe->res != len) {
+		fprintf(stderr, "Bad read amount: %d\n", cqe->res);
+		return 1;
+	}
+	io_uring_cqe_seen(ring, cqe);
+
+	if (memcmp(wvec.iov_base, rvec.iov_base + off, len)) {
+		fprintf(stderr, "%d %d verify failed\n", len, off);
+		return 1;
+	}
+
+	return 0;
+}
+
+static int test(struct io_uring *ring, int fd, int vec_off)
+{
+	struct iovec v = rvec;
+	int ret;
+
+	v.iov_base += vec_off;
+	v.iov_len -= vec_off;
+	ret = io_uring_register_buffers(ring, &v, 1);
+	if (ret) {
+		fprintf(stderr, "Vec register: %d\n", ret);
+		return T_EXIT_FAIL;
+	}
+
+	ret = read_it(ring, fd, 4096, vec_off);
+	if (ret) {
+		fprintf(stderr, "4096 0 failed\n");
+		return T_EXIT_FAIL;
+	}
+	ret = read_it(ring, fd, 8192, 4096);
+	if (ret) {
+		fprintf(stderr, "8192 4096 failed\n");
+		return T_EXIT_FAIL;
+	}
+	ret = read_it(ring, fd, 4096, 4096);
+	if (ret) {
+		fprintf(stderr, "4096 4096 failed\n");
+		return T_EXIT_FAIL;
+	}
+	
+	io_uring_unregister_buffers(ring);
+	return T_EXIT_PASS;
+}
+
+static int is_bdev(int fd)
+{
+	struct stat sb;
+
+	if (fstat(fd, &sb) < 0) {
+		perror("fstat");
+		return -1;
+	}
+
+	return S_ISBLK(sb.st_mode);
+}
+
+int main(int argc, char *argv[])
+{
+	struct io_uring ring;
+	const char *fname;
+	char buf[256];
+	int fd, rnd_fd, ret, bs = 512;
+
+	if (argc > 1) {
+		fname = argv[1];
+	} else {
+		srand((unsigned)time(NULL));
+		snprintf(buf, sizeof(buf), ".fixed-seg-%u-%u", (unsigned) rand(),
+				(unsigned)getpid());
+		fname = buf;
+		t_create_file(fname, 128*1024);
+	}
+
+	fd = open(fname, O_RDWR | O_DIRECT | O_EXCL);
+	if (fd < 0) {
+		perror("open");
+		return 1;
+	}
+	ret = is_bdev(fd);
+	if (fd < 0)
+		return 1;
+	if (ret && ioctl(fd, BLKSSZGET, &bs) < 0) {
+		perror("ioctl BLKSSZGET,");
+		return 1;
+	}
+
+	rnd_fd = open("/dev/urandom", O_RDONLY);
+	if (fd < 0) {
+		perror("urandom: open");
+		goto err;
+	}
+
+	if (posix_memalign(&wvec.iov_base, 4096, 512*1024))
+		goto err;
+	wvec.iov_len = 512*1024;
+
+	ret = read(rnd_fd, wvec.iov_base, wvec.iov_len);
+	if (ret != wvec.iov_len) {
+		fprintf(stderr, "Precondition, urandom read failed, ret: %d\n", ret);
+		goto err;
+	}
+
+	ret = write(fd, wvec.iov_base, wvec.iov_len);
+	if (ret != wvec.iov_len) {
+		fprintf(stderr, "Precondition, write failed, ret: %d\n", ret);
+		goto err;
+	}
+
+	if (posix_memalign(&rvec.iov_base, 4096, 512*1024))
+		goto err;
+	rvec.iov_len = 512*1024;
+
+	ret = io_uring_queue_init(8, &ring, 0);
+	if (ret) {
+		fprintf(stderr, "queue_init: %d\n", ret);
+		goto err;
+	}
+
+	ret = test(&ring, fd, 0);
+	if (ret) {
+		fprintf(stderr, "test 0 failed\n");
+		goto err;
+	}
+
+	ret = test(&ring, fd, bs);
+	if (ret) {
+		fprintf(stderr, "test %d failed\n", bs);
+		goto err;
+	}
+
+	if ((bs == 512) && test(&ring, fd, 3584)) {
+		fprintf(stderr, "test 3584 failed\n");
+		goto err;
+	}
+
+	close(fd);
+	io_uring_queue_exit(&ring);
+	if (fname != argv[1])
+		unlink(fname);
+	return T_EXIT_PASS;
+err:
+	if (fname != argv[1])
+		unlink(fname);
+	return T_EXIT_FAIL;
+}

data/vendor/liburing/test/helpers.c

@@ -363,3 +363,124 @@ unsigned long long utime_since_now(struct timeval *tv)
 	gettimeofday(&end, NULL);
 	return utime_since(tv, &end);
 }
+
+void *aligned_alloc(size_t alignment, size_t size)
+{
+	void *ret;
+
+	if (posix_memalign(&ret, alignment, size))
+		return NULL;
+
+	return ret;
+}
+
+int t_create_socketpair_ip(struct sockaddr_storage *addr,
+				int *sock_client, int *sock_server,
+				bool ipv6, bool client_connect,
+				bool msg_zc, bool tcp, const char *name)
+{
+	socklen_t addr_size;
+	int family, sock, listen_sock = -1;
+	int ret;
+
+	memset(addr, 0, sizeof(*addr));
+	if (ipv6) {
+		struct sockaddr_in6 *saddr = (struct sockaddr_in6 *)addr;
+
+		family = AF_INET6;
+		saddr->sin6_family = family;
+		saddr->sin6_port = htons(0);
+		addr_size = sizeof(*saddr);
+	} else {
+		struct sockaddr_in *saddr = (struct sockaddr_in *)addr;
+
+		family = AF_INET;
+		saddr->sin_family = family;
+		saddr->sin_port = htons(0);
+		saddr->sin_addr.s_addr = htonl(INADDR_ANY);
+		addr_size = sizeof(*saddr);
+	}
+
+	/* server sock setup */
+	if (tcp) {
+		sock = listen_sock = socket(family, SOCK_STREAM, IPPROTO_TCP);
+	} else {
+		sock = *sock_server = socket(family, SOCK_DGRAM, 0);
+	}
+	if (sock < 0) {
+		perror("socket");
+		return 1;
+	}
+
+	ret = bind(sock, (struct sockaddr *)addr, addr_size);
+	if (ret < 0) {
+		perror("bind");
+		return 1;
+	}
+
+	ret = getsockname(sock, (struct sockaddr *)addr, &addr_size);
+	if (ret < 0) {
+		fprintf(stderr, "getsockname failed %i\n", errno);
+		return 1;
+	}
+
+	if (tcp) {
+		ret = listen(sock, 128);
+		assert(ret != -1);
+	}
+
+	if (ipv6) {
+		struct sockaddr_in6 *saddr = (struct sockaddr_in6 *)addr;
+
+		inet_pton(AF_INET6, name, &(saddr->sin6_addr));
+	} else {
+		struct sockaddr_in *saddr = (struct sockaddr_in *)addr;
+
+		inet_pton(AF_INET, name, &saddr->sin_addr);
+	}
+
+	/* client sock setup */
+	if (tcp) {
+		*sock_client = socket(family, SOCK_STREAM, IPPROTO_TCP);
+		assert(client_connect);
+	} else {
+		*sock_client = socket(family, SOCK_DGRAM, 0);
+	}
+	if (*sock_client < 0) {
+		perror("socket");
+		return 1;
+	}
+	if (client_connect) {
+		ret = connect(*sock_client, (struct sockaddr *)addr, addr_size);
+		if (ret < 0) {
+			perror("connect");
+			return 1;
+		}
+	}
+	if (msg_zc) {
+#ifdef SO_ZEROCOPY
+		int val = 1;
+
+		/*
+		 * NOTE: apps must not set SO_ZEROCOPY when using io_uring zc.
+		 * It's only here to test interactions with MSG_ZEROCOPY.
+		 */
+		if (setsockopt(*sock_client, SOL_SOCKET, SO_ZEROCOPY, &val, sizeof(val))) {
+			perror("setsockopt zc");
+			return 1;
+		}
+#else
+		fprintf(stderr, "no SO_ZEROCOPY\n");
+		return 1;
+#endif
+	}
+	if (tcp) {
+		*sock_server = accept(listen_sock, NULL, NULL);
+		if (!*sock_server) {
+			fprintf(stderr, "can't accept\n");
+			return 1;
+		}
+		close(listen_sock);
+	}
+	return 0;
+}

data/vendor/liburing/test/helpers.h

@@ -13,6 +13,7 @@ extern "C" {
 #include "../src/setup.h"
 #include <arpa/inet.h>
 #include <sys/time.h>
+#include <stdlib.h>
 
 enum t_setup_ret {
 	T_SETUP_OK	= 0,
@@ -25,6 +26,13 @@ enum t_test_result {
 	T_EXIT_SKIP   = 77,
 };
 
+/*
+ * Some Android versions lack aligned_alloc in stdlib.h.
+ * To avoid making large changes in tests, define a helper
+ * function that wraps posix_memalign as our own aligned_alloc.
+ */
+void *aligned_alloc(size_t alignment, size_t size);
+
 /*
  * Helper for binding socket to an ephemeral port.
  * The port number to be bound is returned in @addr->sin_port.
@@ -73,6 +81,11 @@ struct iovec *t_create_buffers(size_t buf_num, size_t buf_size);
  */
 int t_create_socket_pair(int fd[2], bool stream);
 
+int t_create_socketpair_ip(struct sockaddr_storage *addr,
+				int *sock_client, int *sock_server,
+				bool ipv6, bool client_connect,
+				bool msg_zc, bool tcp, const char *name);
+
 /*
  * Helper for setting up a ring and checking for user privs
  */

data/vendor/liburing/test/init-mem.c

@@ -41,6 +41,8 @@ static int setup_ctx(struct ctx *ctx, struct q_entries *q)
 	if (posix_memalign(&ctx->mem, 4096, 2*1024*1024))
 		return T_EXIT_FAIL;
 
+	memset(ctx->mem, 0, 2*1024*1024);
+
 	ctx->pre = ctx->mem + 4096 - sizeof(unsigned long long);
 	*ctx->pre = PRE_RED;
 

data/vendor/liburing/test/io_uring_passthrough.c

@@ -13,12 +13,16 @@
 #include "../src/syscall.h"
 #include "nvme.h"
 
+#define min(a, b)	((a) < (b) ? (a) : (b))
+
 #define FILE_SIZE	(256 * 1024)
 #define BS		8192
 #define BUFFERS		(FILE_SIZE / BS)
 
-static struct iovec *vecs;
+static void *meta_mem;
+static struct iovec *vecs, *backing_vec;
 static int no_pt;
+static bool vec_fixed_supported = true;
 
 /*
  * Each offset in the file has the ((test_case / 2) * FILE_SIZE)
@@ -73,7 +77,7 @@ static int __test_io(const char *file, struct io_uring *ring, int tc, int read,
 	struct nvme_uring_cmd *cmd;
 	int open_flags;
 	int do_fixed;
-	int i, ret, fd = -1;
+	int i, ret, fd = -1, use_fd = -1, submit_count = 0;
 	off_t offset;
 	__u64 slba;
 	__u32 nlb;
@@ -84,7 +88,7 @@ static int __test_io(const char *file, struct io_uring *ring, int tc, int read,
 		open_flags = O_WRONLY;
 
 	if (fixed) {
-		ret = t_register_buffers(ring, vecs, BUFFERS);
+		ret = t_register_buffers(ring, backing_vec, 1);
 		if (ret == T_SETUP_SKIP)
 			return 0;
 		if (ret != T_SETUP_OK) {
@@ -114,58 +118,28 @@ static int __test_io(const char *file, struct io_uring *ring, int tc, int read,
 
 	offset = 0;
 	for (i = 0; i < BUFFERS; i++) {
+		unsigned int iovcnt = 1;
+		size_t total_len;
+
 		sqe = io_uring_get_sqe(ring);
 		if (!sqe) {
 			fprintf(stderr, "sqe get failed\n");
 			goto err;
 		}
-		if (read) {
-			int use_fd = fd;
-
-			do_fixed = fixed;
-
-			if (sqthread)
-				use_fd = 0;
-			if (fixed && (i & 1))
-				do_fixed = 0;
-			if (do_fixed) {
-				io_uring_prep_read_fixed(sqe, use_fd, vecs[i].iov_base,
-								vecs[i].iov_len,
-								offset, i);
-				sqe->cmd_op = NVME_URING_CMD_IO;
-			} else if (nonvec) {
-				io_uring_prep_read(sqe, use_fd, vecs[i].iov_base,
-							vecs[i].iov_len, offset);
-				sqe->cmd_op = NVME_URING_CMD_IO;
-			} else {
-				io_uring_prep_readv(sqe, use_fd, &vecs[i], 1,
-								offset);
-				sqe->cmd_op = NVME_URING_CMD_IO_VEC;
-			}
-		} else {
-			int use_fd = fd;
-
-			do_fixed = fixed;
-
-			if (sqthread)
-				use_fd = 0;
-			if (fixed && (i & 1))
-				do_fixed = 0;
-			if (do_fixed) {
-				io_uring_prep_write_fixed(sqe, use_fd, vecs[i].iov_base,
-								vecs[i].iov_len,
-								offset, i);
-				sqe->cmd_op = NVME_URING_CMD_IO;
-			} else if (nonvec) {
-				io_uring_prep_write(sqe, use_fd, vecs[i].iov_base,
-							vecs[i].iov_len, offset);
-				sqe->cmd_op = NVME_URING_CMD_IO;
-			} else {
-				io_uring_prep_writev(sqe, use_fd, &vecs[i], 1,
-								offset);
-				sqe->cmd_op = NVME_URING_CMD_IO_VEC;
-			}
-		}
+		use_fd = fd;
+		do_fixed = fixed;
+
+		if (sqthread)
+			use_fd = 0;
+		if (fixed && (i & 1))
+			do_fixed = 0;
+		if (do_fixed)
+			sqe->buf_index = 0;
+		if (nonvec)
+			sqe->cmd_op = NVME_URING_CMD_IO;
+		else
+			sqe->cmd_op = NVME_URING_CMD_IO_VEC;
+		sqe->fd = use_fd;
 		sqe->opcode = IORING_OP_URING_CMD;
 		if (do_fixed)
 			sqe->uring_cmd_flags |= IORING_URING_CMD_FIXED;
@@ -178,39 +152,58 @@ static int __test_io(const char *file, struct io_uring *ring, int tc, int read,
 
 		cmd->opcode = read ? nvme_cmd_read : nvme_cmd_write;
 
+		if (!nonvec) {
+			iovcnt = (submit_count % 3 == 0) ? 1 : ((submit_count % 3 == 1) ? 3 : 9);
+			iovcnt = min(iovcnt, BUFFERS - i);
+		}
+		total_len = BS * iovcnt;
+
 		slba = offset >> lba_shift;
-		nlb = (BS >> lba_shift) - 1;
+		nlb = (total_len >> lba_shift) - 1;
 
 		/* cdw10 and cdw11 represent starting lba */
 		cmd->cdw10 = slba & 0xffffffff;
 		cmd->cdw11 = slba >> 32;
 		/* cdw12 represent number of lba's for read/write */
 		cmd->cdw12 = nlb;
-		if (do_fixed || nonvec) {
+		if (nonvec) {
 			cmd->addr = (__u64)(uintptr_t)vecs[i].iov_base;
 			cmd->data_len = vecs[i].iov_len;
 		} else {
 			cmd->addr = (__u64)(uintptr_t)&vecs[i];
-			cmd->data_len = 1;
+			cmd->data_len = iovcnt;
+		}
+
+		if (meta_size) {
+			cmd->metadata = (__u64)(uintptr_t)(meta_mem +
+						meta_size * i * (nlb + 1));
+			cmd->metadata_len = meta_size * (nlb + 1);
 		}
 		cmd->nsid = nsid;
 
-		offset += BS;
+		offset += total_len;
+		if (!nonvec)
+			i += iovcnt - 1;
+		submit_count++;
 	}
 
 	ret = io_uring_submit(ring);
-	if (ret != BUFFERS) {
+	if (ret != submit_count) {
 		fprintf(stderr, "submit got %d, wanted %d\n", ret, BUFFERS);
 		goto err;
 	}
 
-	for (i = 0; i < BUFFERS; i++) {
+	for (i = 0; i < submit_count; i++) {
 		ret = io_uring_wait_cqe(ring, &cqe);
 		if (ret) {
 			fprintf(stderr, "wait_cqe=%d\n", ret);
 			goto err;
 		}
 		if (cqe->res != 0) {
+			if (cqe->res == -EINVAL && fixed && !nonvec) {
+				vec_fixed_supported = false;
+				goto cleanup_and_skip;
+			}
 			if (!no_pt) {
 				no_pt = 1;
 				goto skip;
@@ -229,6 +222,7 @@ static int __test_io(const char *file, struct io_uring *ring, int tc, int read,
 		}
 	}
 
+cleanup_and_skip:
 	if (fixed) {
 		ret = io_uring_unregister_buffers(ring);
 		if (ret) {
@@ -254,7 +248,7 @@ err:
 }
 
 static int test_io(const char *file, int tc, int read, int sqthread,
-		   int fixed, int nonvec)
+		   int fixed, int nonvec, int hybrid)
 {
 	struct io_uring ring;
 	int ret, ring_flags = 0;
@@ -265,6 +259,12 @@ static int test_io(const char *file, int tc, int read, int sqthread,
 	if (sqthread)
 		ring_flags |= IORING_SETUP_SQPOLL;
 
+	if (hybrid)
+		ring_flags |= IORING_SETUP_IOPOLL | IORING_SETUP_HYBRID_IOPOLL;
+
+	if (fixed && (!vec_fixed_supported && !nonvec))
+		return 0;
+
 	ret = t_create_ring(64, &ring, ring_flags);
 	if (ret == T_SETUP_SKIP)
 		return 0;
@@ -401,6 +401,12 @@ static int test_io_uring_submit_enters(const char *file)
 		cmd->addr = (__u64)(uintptr_t)&vecs[i];
 		cmd->data_len = 1;
 		cmd->nsid = nsid;
+
+		if (meta_size) {
+			cmd->metadata = (__u64)(uintptr_t)(meta_mem +
+						meta_size * i * (nlb + 1));
+			cmd->metadata_len = meta_size * (nlb + 1);
+		}
 	}
 
 	/* submit manually to avoid adding IORING_ENTER_GETEVENTS */
@@ -447,20 +453,30 @@ int main(int argc, char *argv[])
 	if (ret)
 		return T_EXIT_SKIP;
 
-	vecs = t_create_buffers(BUFFERS, BS);
+	vecs = t_malloc(BUFFERS * sizeof(struct iovec));
+	backing_vec = t_create_buffers(1, BUFFERS * BS);
+	/* Slice single large backing_vec into multiple smaller vecs */
+	for (int i = 0; i < BUFFERS; i++) {
+		vecs[i].iov_base = backing_vec[0].iov_base + i * BS;
+		vecs[i].iov_len = BS;
+	}
+	if (meta_size)
+		t_posix_memalign(&meta_mem, 0x1000,
+				 meta_size * BUFFERS * (BS >> lba_shift));
 
-	for (i = 0; i < 16; i++) {
+	for (i = 0; i < 32; i++) {
 		int read = (i & 1) != 0;
 		int sqthread = (i & 2) != 0;
 		int fixed = (i & 4) != 0;
 		int nonvec = (i & 8) != 0;
+		int hybrid = (i & 16) != 0;
 
-		ret = test_io(fname, i, read, sqthread, fixed, nonvec);
+		ret = test_io(fname, i, read, sqthread, fixed, nonvec, hybrid);
 		if (no_pt)
 			break;
 		if (ret) {
-			fprintf(stderr, "test_io failed %d/%d/%d/%d\n",
-				read, sqthread, fixed, nonvec);
+			fprintf(stderr, "test_io failed %d/%d/%d/%d/%d\n",
+				read, sqthread, fixed, nonvec, hybrid);
 			goto err;
 		}
 	}

data/vendor/liburing/test/iopoll-overflow.c

@@ -73,14 +73,15 @@ int main(int argc, char *argv[])
 	char *fname;
 	int ret, fd;
 
-	p.flags = IORING_SETUP_IOPOLL | IORING_SETUP_CQSIZE;
+	p.flags = IORING_SETUP_IOPOLL | IORING_SETUP_CQSIZE |
+			IORING_SETUP_SUBMIT_ALL;
 	p.cq_entries = 64;
 	ret = t_create_ring_params(64, &ring, &p);
-	if (ret == T_SETUP_SKIP)
-		return 0;
+	if (ret == T_SETUP_SKIP || ret == -EINVAL)
+		return T_EXIT_SKIP;
 	if (ret != T_SETUP_OK) {
 		fprintf(stderr, "ring create failed: %d\n", ret);
-		return 1;
+		return T_EXIT_FAIL;
 	}
 
 	if (argc > 1) {

data/vendor/liburing/test/iopoll.c

@@ -23,6 +23,7 @@
 static struct iovec *vecs;
 static int no_buf_select;
 static int no_iopoll;
+static int no_hybrid;
 
 static int provide_buffers(struct io_uring *ring)
 {
@@ -351,7 +352,7 @@ ok:
 }
 
 static int test_io(const char *file, int write, int sqthread, int fixed,
-		   int buf_select, int defer)
+		   int hybrid, int buf_select, int defer)
 {
 	struct io_uring ring;
 	int ret, ring_flags = IORING_SETUP_IOPOLL;
@@ -363,10 +364,18 @@ static int test_io(const char *file, int write, int sqthread, int fixed,
 		ring_flags |= IORING_SETUP_SINGLE_ISSUER |
 			      IORING_SETUP_DEFER_TASKRUN;
 
+	if (hybrid)
+		ring_flags |= IORING_SETUP_HYBRID_IOPOLL;
+
 	ret = t_create_ring(64, &ring, ring_flags);
-	if (ret == T_SETUP_SKIP)
+	if (ret == T_SETUP_SKIP) {
 		return 0;
+	}
 	if (ret != T_SETUP_OK) {
+		if (ring_flags & IORING_SETUP_HYBRID_IOPOLL) {
+			no_hybrid = 1;
+			return 0;
+		}
 		fprintf(stderr, "ring create failed: %d\n", ret);
 		return 1;
 	}
@@ -418,22 +427,23 @@ int main(int argc, char *argv[])
 
 	vecs = t_create_buffers(BUFFERS, BS);
 
-	nr = 32;
+	nr = 64;
 	if (no_buf_select)
-		nr = 8;
-	else if (!t_probe_defer_taskrun())
 		nr = 16;
+	else if (!t_probe_defer_taskrun())
+		nr = 32;
 	for (i = 0; i < nr; i++) {
 		int write = (i & 1) != 0;
 		int sqthread = (i & 2) != 0;
 		int fixed = (i & 4) != 0;
-		int buf_select = (i & 8) != 0;
-		int defer = (i & 16) != 0;
+		int hybrid = (i & 8) != 0;
+		int buf_select = (i & 16) != 0;
+		int defer = (i & 32) != 0;
 
-		ret = test_io(fname, write, sqthread, fixed, buf_select, defer);
+		ret = test_io(fname, write, sqthread, fixed, hybrid, buf_select, defer);
 		if (ret) {
-			fprintf(stderr, "test_io failed %d/%d/%d/%d/%d\n",
-				write, sqthread, fixed, buf_select, defer);
+			fprintf(stderr, "test_io failed %d/%d/%d/%d/%d/%d\n",
+				write, sqthread, fixed, hybrid, buf_select, defer);
 			goto err;
 		}
 		if (no_iopoll)