polyphony 0.83 → 0.84

data/ext/liburing/queue.c

@@ -1,333 +0,0 @@
-/* SPDX-License-Identifier: MIT */
-#include <sys/types.h>
-#include <sys/stat.h>
-#include <sys/mman.h>
-#include <unistd.h>
-#include <errno.h>
-#include <string.h>
-#include <stdbool.h>
-
-#include "liburing/compat.h"
-#include "liburing/io_uring.h"
-#include "liburing.h"
-#include "liburing/barrier.h"
-
-#include "syscall.h"
-
-/*
- * Returns true if we're not using SQ thread (thus nobody submits but us)
- * or if IORING_SQ_NEED_WAKEUP is set, so submit thread must be explicitly
- * awakened. For the latter case, we set the thread wakeup flag.
- */
-static inline bool sq_ring_needs_enter(struct io_uring *ring, unsigned *flags)
-{
-	if (!(ring->flags & IORING_SETUP_SQPOLL))
-		return true;
-	if (IO_URING_READ_ONCE(*ring->sq.kflags) & IORING_SQ_NEED_WAKEUP) {
-		*flags |= IORING_ENTER_SQ_WAKEUP;
-		return true;
-	}
-
-	return false;
-}
-
-static inline bool cq_ring_needs_flush(struct io_uring *ring)
-{
-	return IO_URING_READ_ONCE(*ring->sq.kflags) & IORING_SQ_CQ_OVERFLOW;
-}
-
-static int __io_uring_peek_cqe(struct io_uring *ring,
-			       struct io_uring_cqe **cqe_ptr)
-{
-	struct io_uring_cqe *cqe;
-	unsigned head;
-	int err = 0;
-
-	do {
-		io_uring_for_each_cqe(ring, head, cqe)
-			break;
-		if (cqe) {
-			if (cqe->user_data == LIBURING_UDATA_TIMEOUT) {
-				if (cqe->res < 0)
-					err = cqe->res;
-				io_uring_cq_advance(ring, 1);
-				if (!err)
-					continue;
-				cqe = NULL;
-			}
-		}
-		break;
-	} while (1);
-
-	*cqe_ptr = cqe;
-	return err;
-}
-
-int __io_uring_get_cqe(struct io_uring *ring, struct io_uring_cqe **cqe_ptr,
-		       unsigned submit, unsigned wait_nr, sigset_t *sigmask)
-{
-	struct io_uring_cqe *cqe = NULL;
-	const int to_wait = wait_nr;
-	int ret = 0, err;
-
-	do {
-		bool cq_overflow_flush = false;
-		unsigned flags = 0;
-
-		err = __io_uring_peek_cqe(ring, &cqe);
-		if (err)
-			break;
-		if (!cqe && !to_wait && !submit) {
-			if (!cq_ring_needs_flush(ring)) {
-				err = -EAGAIN;
-				break;
-			}
-			cq_overflow_flush = true;
-		}
-		if (wait_nr && cqe)
-			wait_nr--;
-		if (wait_nr || cq_overflow_flush)
-			flags = IORING_ENTER_GETEVENTS;
-		if (submit)
-			sq_ring_needs_enter(ring, &flags);
-		if (wait_nr || submit || cq_overflow_flush)
-			ret = __sys_io_uring_enter(ring->ring_fd, submit,
-						   wait_nr, flags, sigmask);
-		if (ret < 0) {
-			err = -errno;
-		} else if (ret == (int)submit) {
-			submit = 0;
-			/*
-			 * When SETUP_IOPOLL is set, __sys_io_uring enter()
-			 * must be called to reap new completions but the call
-			 * won't be made if both wait_nr and submit are zero
-			 * so preserve wait_nr.
-			 */
-			if (!(ring->flags & IORING_SETUP_IOPOLL))
-				wait_nr = 0;
-		} else {
-			submit -= ret;
-		}
-		if (cqe)
-			break;
-	} while (!err);
-
-	*cqe_ptr = cqe;
-	return err;
-}
-
-/*
- * Fill in an array of IO completions up to count, if any are available.
- * Returns the amount of IO completions filled.
- */
-unsigned io_uring_peek_batch_cqe(struct io_uring *ring,
-				 struct io_uring_cqe **cqes, unsigned count)
-{
-	unsigned ready;
-	bool overflow_checked = false;
-
-again:
-	ready = io_uring_cq_ready(ring);
-	if (ready) {
-		unsigned head = *ring->cq.khead;
-		unsigned mask = *ring->cq.kring_mask;
-		unsigned last;
-		int i = 0;
-
-		count = count > ready ? ready : count;
-		last = head + count;
-		for (;head != last; head++, i++)
-			cqes[i] = &ring->cq.cqes[head & mask];
-
-		return count;
-	}
-
-	if (overflow_checked)
-		goto done;
-
-	if (cq_ring_needs_flush(ring)) {
-		__sys_io_uring_enter(ring->ring_fd, 0, 0,
-				     IORING_ENTER_GETEVENTS, NULL);
-		overflow_checked = true;
-		goto again;
-	}
-
-done:
-	return 0;
-}
-
-/*
- * Sync internal state with kernel ring state on the SQ side. Returns the
- * number of pending items in the SQ ring, for the shared ring.
- */
-int __io_uring_flush_sq(struct io_uring *ring)
-{
-	struct io_uring_sq *sq = &ring->sq;
-	const unsigned mask = *sq->kring_mask;
-	unsigned ktail, to_submit;
-
-	if (sq->sqe_head == sq->sqe_tail) {
-		ktail = *sq->ktail;
-		goto out;
-	}
-
-	/*
-	 * Fill in sqes that we have queued up, adding them to the kernel ring
-	 */
-	ktail = *sq->ktail;
-	to_submit = sq->sqe_tail - sq->sqe_head;
-	while (to_submit--) {
-		sq->array[ktail & mask] = sq->sqe_head & mask;
-		ktail++;
-		sq->sqe_head++;
-	}
-
-	/*
-	 * Ensure that the kernel sees the SQE updates before it sees the tail
-	 * update.
-	 */
-	io_uring_smp_store_release(sq->ktail, ktail);
-out:
-	return ktail - *sq->khead;
-}
-
-/*
- * Like io_uring_wait_cqe(), except it accepts a timeout value as well. Note
- * that an sqe is used internally to handle the timeout. Applications using
- * this function must never set sqe->user_data to LIBURING_UDATA_TIMEOUT!
- *
- * If 'ts' is specified, the application need not call io_uring_submit() before
- * calling this function, as we will do that on its behalf. From this it also
- * follows that this function isn't safe to use for applications that split SQ
- * and CQ handling between two threads and expect that to work without
- * synchronization, as this function manipulates both the SQ and CQ side.
- */
-int io_uring_wait_cqes(struct io_uring *ring, struct io_uring_cqe **cqe_ptr,
-		       unsigned wait_nr, struct __kernel_timespec *ts,
-		       sigset_t *sigmask)
-{
-	unsigned to_submit = 0;
-
-	if (ts) {
-		struct io_uring_sqe *sqe;
-		int ret;
-
-		/*
-		 * If the SQ ring is full, we may need to submit IO first
-		 */
-		sqe = io_uring_get_sqe(ring);
-		if (!sqe) {
-			ret = io_uring_submit(ring);
-			if (ret < 0)
-				return ret;
-			sqe = io_uring_get_sqe(ring);
-			if (!sqe)
-				return -EAGAIN;
-		}
-		io_uring_prep_timeout(sqe, ts, wait_nr, 0);
-		sqe->user_data = LIBURING_UDATA_TIMEOUT;
-		to_submit = __io_uring_flush_sq(ring);
-	}
-
-	return __io_uring_get_cqe(ring, cqe_ptr, to_submit, wait_nr, sigmask);
-}
-
-/*
- * See io_uring_wait_cqes() - this function is the same, it just always uses
- * '1' as the wait_nr.
- */
-int io_uring_wait_cqe_timeout(struct io_uring *ring,
-			      struct io_uring_cqe **cqe_ptr,
-			      struct __kernel_timespec *ts)
-{
-	return io_uring_wait_cqes(ring, cqe_ptr, 1, ts, NULL);
-}
-
-/*
- * Submit sqes acquired from io_uring_get_sqe() to the kernel.
- *
- * Returns number of sqes submitted
- */
-static int __io_uring_submit(struct io_uring *ring, unsigned submitted,
-			     unsigned wait_nr)
-{
-	unsigned flags;
-	int ret;
-
-	flags = 0;
-	if (sq_ring_needs_enter(ring, &flags) || wait_nr) {
-		if (wait_nr || (ring->flags & IORING_SETUP_IOPOLL))
-			flags |= IORING_ENTER_GETEVENTS;
-
-		ret = __sys_io_uring_enter(ring->ring_fd, submitted, wait_nr,
-						flags, NULL);
-		if (ret < 0)
-			return -errno;
-	} else
-		ret = submitted;
-
-	return ret;
-}
-
-static int __io_uring_submit_and_wait(struct io_uring *ring, unsigned wait_nr)
-{
-	return __io_uring_submit(ring, __io_uring_flush_sq(ring), wait_nr);
-}
-
-/*
- * Submit sqes acquired from io_uring_get_sqe() to the kernel.
- *
- * Returns number of sqes submitted
- */
-int io_uring_submit(struct io_uring *ring)
-{
-	return __io_uring_submit_and_wait(ring, 0);
-}
-
-/*
- * Like io_uring_submit(), but allows waiting for events as well.
- *
- * Returns number of sqes submitted
- */
-int io_uring_submit_and_wait(struct io_uring *ring, unsigned wait_nr)
-{
-	return __io_uring_submit_and_wait(ring, wait_nr);
-}
-
-static inline struct io_uring_sqe *
-__io_uring_get_sqe(struct io_uring_sq *sq, unsigned int __head)
-{
-	unsigned int __next = (sq)->sqe_tail + 1;
-	struct io_uring_sqe *__sqe = NULL;
-
-	if (__next - __head <= *(sq)->kring_entries) {
-		__sqe = &(sq)->sqes[(sq)->sqe_tail & *(sq)->kring_mask];
-		(sq)->sqe_tail = __next;
-	}
-	return __sqe;
-}
-
-/*
- * Return an sqe to fill. Application must later call io_uring_submit()
- * when it's ready to tell the kernel about it. The caller may call this
- * function multiple times before calling io_uring_submit().
- *
- * Returns a vacant sqe, or NULL if we're full.
- */
-struct io_uring_sqe *io_uring_get_sqe(struct io_uring *ring)
-{
-	struct io_uring_sq *sq = &ring->sq;
-
-	return __io_uring_get_sqe(sq, io_uring_smp_load_acquire(sq->khead));
-}
-
-int __io_uring_sqring_wait(struct io_uring *ring)
-{
-	int ret;
-
-	ret = __sys_io_uring_enter(ring->ring_fd, 0, 0, IORING_ENTER_SQ_WAIT,
-					NULL);
-	if (ret < 0)
-		ret = -errno;
-	return ret;
-}

data/ext/liburing/register.c

@@ -1,187 +0,0 @@
-/* SPDX-License-Identifier: MIT */
-#include <sys/types.h>
-#include <sys/stat.h>
-#include <sys/mman.h>
-#include <unistd.h>
-#include <errno.h>
-#include <string.h>
-
-#include "liburing/compat.h"
-#include "liburing/io_uring.h"
-#include "liburing.h"
-
-#include "syscall.h"
-
-int io_uring_register_buffers(struct io_uring *ring, const struct iovec *iovecs,
-			      unsigned nr_iovecs)
-{
-	int ret;
-
-	ret = __sys_io_uring_register(ring->ring_fd, IORING_REGISTER_BUFFERS,
-					iovecs, nr_iovecs);
-	if (ret < 0)
-		return -errno;
-
-	return 0;
-}
-
-int io_uring_unregister_buffers(struct io_uring *ring)
-{
-	int ret;
-
-	ret = __sys_io_uring_register(ring->ring_fd, IORING_UNREGISTER_BUFFERS,
-					NULL, 0);
-	if (ret < 0)
-		return -errno;
-
-	return 0;
-}
-
-/*
- * Register an update for an existing file set. The updates will start at
- * 'off' in the original array, and 'nr_files' is the number of files we'll
- * update.
- *
- * Returns number of files updated on success, -ERROR on failure.
- */
-int io_uring_register_files_update(struct io_uring *ring, unsigned off,
-				   int *files, unsigned nr_files)
-{
-	struct io_uring_files_update up = {
-		.offset	= off,
-		.fds	= (unsigned long) files,
-	};
-	int ret;
-
-	ret = __sys_io_uring_register(ring->ring_fd,
-					IORING_REGISTER_FILES_UPDATE, &up,
-					nr_files);
-	if (ret < 0)
-		return -errno;
-
-	return ret;
-}
-
-int io_uring_register_files(struct io_uring *ring, const int *files,
-			      unsigned nr_files)
-{
-	int ret;
-
-	ret = __sys_io_uring_register(ring->ring_fd, IORING_REGISTER_FILES,
-					files, nr_files);
-	if (ret < 0)
-		return -errno;
-
-	return 0;
-}
-
-int io_uring_unregister_files(struct io_uring *ring)
-{
-	int ret;
-
-	ret = __sys_io_uring_register(ring->ring_fd, IORING_UNREGISTER_FILES,
-					NULL, 0);
-	if (ret < 0)
-		return -errno;
-
-	return 0;
-}
-
-int io_uring_register_eventfd(struct io_uring *ring, int event_fd)
-{
-	int ret;
-
-	ret = __sys_io_uring_register(ring->ring_fd, IORING_REGISTER_EVENTFD,
-					&event_fd, 1);
-	if (ret < 0)
-		return -errno;
-
-	return 0;
-}
-
-int io_uring_unregister_eventfd(struct io_uring *ring)
-{
-	int ret;
-
-	ret = __sys_io_uring_register(ring->ring_fd, IORING_UNREGISTER_EVENTFD,
-					NULL, 0);
-	if (ret < 0)
-		return -errno;
-
-	return 0;
-}
-
-int io_uring_register_eventfd_async(struct io_uring *ring, int event_fd)
-{
-	int ret;
-
-	ret = __sys_io_uring_register(ring->ring_fd, IORING_REGISTER_EVENTFD_ASYNC,
-			&event_fd, 1);
-	if (ret < 0)
-		return -errno;
-
-	return 0;
-}
-
-int io_uring_register_probe(struct io_uring *ring, struct io_uring_probe *p,
-			    unsigned int nr_ops)
-{
-	int ret;
-
-	ret = __sys_io_uring_register(ring->ring_fd, IORING_REGISTER_PROBE,
-					p, nr_ops);
-	if (ret < 0)
-		return -errno;
-
-	return 0;
-}
-
-int io_uring_register_personality(struct io_uring *ring)
-{
-	int ret;
-
-	ret = __sys_io_uring_register(ring->ring_fd, IORING_REGISTER_PERSONALITY,
-					NULL, 0);
-	if (ret < 0)
-		return -errno;
-
-	return ret;
-}
-
-int io_uring_unregister_personality(struct io_uring *ring, int id)
-{
-	int ret;
-
-	ret = __sys_io_uring_register(ring->ring_fd, IORING_UNREGISTER_PERSONALITY,
-					NULL, id);
-	if (ret < 0)
-		return -errno;
-
-	return ret;
-}
-
-int io_uring_register_restrictions(struct io_uring *ring,
-				   struct io_uring_restriction *res,
-				   unsigned int nr_res)
-{
-	int ret;
-
-	ret = __sys_io_uring_register(ring->ring_fd, IORING_REGISTER_RESTRICTIONS,
-				      res, nr_res);
-	if (ret < 0)
-		return -errno;
-
-	return 0;
-}
-
-int io_uring_enable_rings(struct io_uring *ring)
-{
-	int ret;
-
-	ret = __sys_io_uring_register(ring->ring_fd,
-				      IORING_REGISTER_ENABLE_RINGS, NULL, 0);
-	if (ret < 0)
-		return -errno;
-
-	return ret;
-}

data/ext/liburing/setup.c

@@ -1,210 +0,0 @@
-/* SPDX-License-Identifier: MIT */
-#include <sys/types.h>
-#include <sys/stat.h>
-#include <sys/mman.h>
-#include <unistd.h>
-#include <errno.h>
-#include <string.h>
-#include <stdlib.h>
-
-#include "liburing/compat.h"
-#include "liburing/io_uring.h"
-#include "liburing.h"
-
-#include "syscall.h"
-
-static void io_uring_unmap_rings(struct io_uring_sq *sq, struct io_uring_cq *cq)
-{
-	munmap(sq->ring_ptr, sq->ring_sz);
-	if (cq->ring_ptr && cq->ring_ptr != sq->ring_ptr)
-		munmap(cq->ring_ptr, cq->ring_sz);
-}
-
-static int io_uring_mmap(int fd, struct io_uring_params *p,
-			 struct io_uring_sq *sq, struct io_uring_cq *cq)
-{
-	size_t size;
-	int ret;
-
-	sq->ring_sz = p->sq_off.array + p->sq_entries * sizeof(unsigned);
-	cq->ring_sz = p->cq_off.cqes + p->cq_entries * sizeof(struct io_uring_cqe);
-
-	if (p->features & IORING_FEAT_SINGLE_MMAP) {
-		if (cq->ring_sz > sq->ring_sz)
-			sq->ring_sz = cq->ring_sz;
-		cq->ring_sz = sq->ring_sz;
-	}
-	sq->ring_ptr = mmap(0, sq->ring_sz, PROT_READ | PROT_WRITE,
-			MAP_SHARED | MAP_POPULATE, fd, IORING_OFF_SQ_RING);
-	if (sq->ring_ptr == MAP_FAILED)
-		return -errno;
-
-	if (p->features & IORING_FEAT_SINGLE_MMAP) {
-		cq->ring_ptr = sq->ring_ptr;
-	} else {
-		cq->ring_ptr = mmap(0, cq->ring_sz, PROT_READ | PROT_WRITE,
-				MAP_SHARED | MAP_POPULATE, fd, IORING_OFF_CQ_RING);
-		if (cq->ring_ptr == MAP_FAILED) {
-			cq->ring_ptr = NULL;
-			ret = -errno;
-			goto err;
-		}
-	}
-
-	sq->khead = sq->ring_ptr + p->sq_off.head;
-	sq->ktail = sq->ring_ptr + p->sq_off.tail;
-	sq->kring_mask = sq->ring_ptr + p->sq_off.ring_mask;
-	sq->kring_entries = sq->ring_ptr + p->sq_off.ring_entries;
-	sq->kflags = sq->ring_ptr + p->sq_off.flags;
-	sq->kdropped = sq->ring_ptr + p->sq_off.dropped;
-	sq->array = sq->ring_ptr + p->sq_off.array;
-
-	size = p->sq_entries * sizeof(struct io_uring_sqe);
-	sq->sqes = mmap(0, size, PROT_READ | PROT_WRITE,
-				MAP_SHARED | MAP_POPULATE, fd,
-				IORING_OFF_SQES);
-	if (sq->sqes == MAP_FAILED) {
-		ret = -errno;
-err:
-		io_uring_unmap_rings(sq, cq);
-		return ret;
-	}
-
-	cq->khead = cq->ring_ptr + p->cq_off.head;
-	cq->ktail = cq->ring_ptr + p->cq_off.tail;
-	cq->kring_mask = cq->ring_ptr + p->cq_off.ring_mask;
-	cq->kring_entries = cq->ring_ptr + p->cq_off.ring_entries;
-	cq->koverflow = cq->ring_ptr + p->cq_off.overflow;
-	cq->cqes = cq->ring_ptr + p->cq_off.cqes;
-	if (p->cq_off.flags)
-		cq->kflags = cq->ring_ptr + p->cq_off.flags;
-	return 0;
-}
-
-/*
- * For users that want to specify sq_thread_cpu or sq_thread_idle, this
- * interface is a convenient helper for mmap()ing the rings.
- * Returns -errno on error, or zero on success.  On success, 'ring'
- * contains the necessary information to read/write to the rings.
- */
-int io_uring_queue_mmap(int fd, struct io_uring_params *p, struct io_uring *ring)
-{
-	int ret;
-
-	memset(ring, 0, sizeof(*ring));
-	ret = io_uring_mmap(fd, p, &ring->sq, &ring->cq);
-	if (!ret) {
-		ring->flags = p->flags;
-		ring->ring_fd = fd;
-	}
-	return ret;
-}
-
-/*
- * Ensure that the mmap'ed rings aren't available to a child after a fork(2).
- * This uses madvise(..., MADV_DONTFORK) on the mmap'ed ranges.
- */
-int io_uring_ring_dontfork(struct io_uring *ring)
-{
-	size_t len;
-	int ret;
-
-	if (!ring->sq.ring_ptr || !ring->sq.sqes || !ring->cq.ring_ptr)
-		return -EINVAL;
-
-	len = *ring->sq.kring_entries * sizeof(struct io_uring_sqe);
-	ret = madvise(ring->sq.sqes, len, MADV_DONTFORK);
-	if (ret == -1)
-		return -errno;
-
-	len = ring->sq.ring_sz;
-	ret = madvise(ring->sq.ring_ptr, len, MADV_DONTFORK);
-	if (ret == -1)
-		return -errno;
-
-	if (ring->cq.ring_ptr != ring->sq.ring_ptr) {
-		len = ring->cq.ring_sz;
-		ret = madvise(ring->cq.ring_ptr, len, MADV_DONTFORK);
-		if (ret == -1)
-			return -errno;
-	}
-
-	return 0;
-}
-
-int io_uring_queue_init_params(unsigned entries, struct io_uring *ring,
-			       struct io_uring_params *p)
-{
-	int fd, ret;
-
-	fd = __sys_io_uring_setup(entries, p);
-	if (fd < 0)
-		return -errno;
-
-	ret = io_uring_queue_mmap(fd, p, ring);
-	if (ret)
-		close(fd);
-
-	return ret;
-}
-
-/*
- * Returns -errno on error, or zero on success. On success, 'ring'
- * contains the necessary information to read/write to the rings.
- */
-int io_uring_queue_init(unsigned entries, struct io_uring *ring, unsigned flags)
-{
-	struct io_uring_params p;
-
-	memset(&p, 0, sizeof(p));
-	p.flags = flags;
-
-	return io_uring_queue_init_params(entries, ring, &p);
-}
-
-void io_uring_queue_exit(struct io_uring *ring)
-{
-	struct io_uring_sq *sq = &ring->sq;
-	struct io_uring_cq *cq = &ring->cq;
-
-	munmap(sq->sqes, *sq->kring_entries * sizeof(struct io_uring_sqe));
-	io_uring_unmap_rings(sq, cq);
-	close(ring->ring_fd);
-}
-
-struct io_uring_probe *io_uring_get_probe_ring(struct io_uring *ring)
-{
-	struct io_uring_probe *probe;
-	int r;
-
-	size_t len = sizeof(*probe) + 256 * sizeof(struct io_uring_probe_op);
-	probe = malloc(len);
-	memset(probe, 0, len);
-	r = io_uring_register_probe(ring, probe, 256);
-	if (r < 0)
-		goto fail;
-
-	return probe;
-fail:
-	free(probe);
-	return NULL;
-}
-
-struct io_uring_probe *io_uring_get_probe(void)
-{
-	struct io_uring ring;
-	struct io_uring_probe* probe = NULL;
-
-	int r = io_uring_queue_init(2, &ring, 0);
-	if (r < 0)
-		return NULL;
-
-	probe = io_uring_get_probe_ring(&ring);
-	io_uring_queue_exit(&ring);
-	return probe;
-}
-
-void io_uring_free_probe(struct io_uring_probe *probe)
-{
-	free(probe);
-}