io-event 1.16.2 → 1.21.1

data/ext/io/event/selector/uring.c

@@ -11,13 +11,21 @@
 #include <stdbool.h>
 #include <stdint.h>
 #include <time.h>
+#include <unistd.h>
 
 #include "../interrupt.h"
 
-#include "pidfd.c"
-
 #include <linux/version.h>
 
+// `io_uring` support for `IORING_OP_WAITID` was introduced in Linux 6.7. When available, we use it to wait for process exit directly in the ring, instead of polling on a pidfd.
+#if defined(HAVE_IO_URING_PREP_WAITID) && (LINUX_VERSION_CODE >= KERNEL_VERSION(6,7,0))
+#define IO_EVENT_SELECTOR_URING_USE_WAITID
+#endif
+
+#ifndef IO_EVENT_SELECTOR_URING_USE_WAITID
+#include "pidfd.c"
+#endif
+
 enum {
 	DEBUG = 0,
 	DEBUG_COMPLETION = 0,
@@ -32,10 +40,7 @@ struct IO_Event_Selector_URing
 {
 	struct IO_Event_Selector backend;
 	struct io_uring ring;
-	
-	// Flag indicating whether the selector is currently blocked in a system call.
-	// Set to 1 when blocked in io_uring_wait_cqe_timeout() without GVL, 0 otherwise.
-	int blocked;
+	pid_t owner;
 	
 	// Interrupt used to wake the selector from another thread without touching the ring's SQ.
 	// This allows IORING_SETUP_SINGLE_ISSUER: only the owner thread ever submits SQEs.
@@ -76,8 +81,27 @@ struct IO_Event_Selector_URing_Completion
 	struct IO_Event_List list;
 	
 	struct IO_Event_Selector_URing_Waiting *waiting;
+	bool operation_pending;
+	bool cancellation_pending;
 };
 
+// Cancellation completions use the low bit to distinguish them from the
+// completion of the operation being cancelled. Completion records are aligned,
+// so the low bit is otherwise always clear.
+#define IO_EVENT_SELECTOR_URING_CANCELLATION_TAG ((uint64_t)1)
+
+static inline uint64_t
+IO_Event_Selector_URing_Completion_cancellation_data(struct IO_Event_Selector_URing_Completion *completion)
+{
+	return ((uint64_t)(uintptr_t)completion) | IO_EVENT_SELECTOR_URING_CANCELLATION_TAG;
+}
+
+static inline struct IO_Event_Selector_URing_Completion *
+IO_Event_Selector_URing_Completion_from_cancellation_data(uint64_t data)
+{
+	return (struct IO_Event_Selector_URing_Completion *)(uintptr_t)(data & ~IO_EVENT_SELECTOR_URING_CANCELLATION_TAG);
+}
+
 static
 void IO_Event_Selector_URing_Completion_mark(void *_completion)
 {
@@ -115,14 +139,20 @@ void IO_Event_Selector_URing_Type_compact(void *_selector)
 static
 void close_internal(struct IO_Event_Selector_URing *selector)
 {
-	if (selector->interrupt.descriptor >= 0) {
-		IO_Event_Interrupt_close(&selector->interrupt);
+	if (selector->owner == getpid()) {
+		if (selector->interrupt.descriptor >= 0) {
+			IO_Event_Interrupt_close(&selector->interrupt);
+			selector->interrupt.descriptor = -1;
+			selector->wakeup_registered = 0;
+		}
+		
+		if (selector->ring.ring_fd >= 0) {
+			io_uring_queue_exit(&selector->ring);
+			selector->ring.ring_fd = -1;
+		}
+	} else {
 		selector->interrupt.descriptor = -1;
 		selector->wakeup_registered = 0;
-	}
-	
-	if (selector->ring.ring_fd >= 0) {
-		io_uring_queue_exit(&selector->ring);
 		selector->ring.ring_fd = -1;
 	}
 }
@@ -177,8 +207,13 @@ struct IO_Event_Selector_URing_Completion * IO_Event_Selector_URing_Completion_a
 	
 	if (DEBUG_COMPLETION) fprintf(stderr, "IO_Event_Selector_URing_Completion_acquire(%p, limit=%ld)\n", (void*)completion, selector->completions.limit);
 	
+	assert(completion->waiting == NULL);
+	assert(!completion->operation_pending);
+	assert(!completion->cancellation_pending);
+	
 	waiting->completion = completion;
 	completion->waiting = waiting;
+	completion->operation_pending = true;
 	
 	return completion;
 }
@@ -195,14 +230,45 @@ void IO_Event_Selector_URing_Completion_cancel(struct IO_Event_Selector_URing_Co
 }
 
 inline static
-void IO_Event_Selector_URing_Completion_release(struct IO_Event_Selector_URing *selector, struct IO_Event_Selector_URing_Completion *completion)
+void IO_Event_Selector_URing_Completion_recycle(struct IO_Event_Selector_URing *selector, struct IO_Event_Selector_URing_Completion *completion)
 {
-	if (DEBUG_COMPLETION) fprintf(stderr, "IO_Event_Selector_URing_Completion_release(%p)\n", (void*)completion);
+	assert(completion->waiting == NULL);
+	assert(!completion->operation_pending);
+	assert(!completion->cancellation_pending);
 	
-	IO_Event_Selector_URing_Completion_cancel(completion);
 	IO_Event_List_prepend(&selector->free_list, &completion->list);
 }
 
+inline static
+void IO_Event_Selector_URing_Completion_complete(struct IO_Event_Selector_URing *selector, struct IO_Event_Selector_URing_Completion *completion)
+{
+	if (DEBUG_COMPLETION) fprintf(stderr, "IO_Event_Selector_URing_Completion_complete(%p)\n", (void*)completion);
+	
+	assert(completion->operation_pending);
+	completion->operation_pending = false;
+	
+	IO_Event_Selector_URing_Completion_cancel(completion);
+	
+	// A cancellation SQE still refers to this completion record. Keep it out of
+	// the free list until that CQE has also been observed, avoiding ABA reuse.
+	if (!completion->cancellation_pending) {
+		IO_Event_Selector_URing_Completion_recycle(selector, completion);
+	}
+}
+
+inline static
+void IO_Event_Selector_URing_Completion_cancellation_complete(struct IO_Event_Selector_URing *selector, struct IO_Event_Selector_URing_Completion *completion)
+{
+	if (DEBUG_COMPLETION) fprintf(stderr, "IO_Event_Selector_URing_Completion_cancellation_complete(%p)\n", (void*)completion);
+	
+	assert(completion->cancellation_pending);
+	completion->cancellation_pending = false;
+	
+	if (!completion->operation_pending) {
+		IO_Event_Selector_URing_Completion_recycle(selector, completion);
+	}
+}
+
 inline static
 void IO_Event_Selector_URing_Waiting_cancel(struct IO_Event_Selector_URing_Waiting *waiting)
 {
@@ -223,6 +289,9 @@ void IO_Event_Selector_URing_Completion_initialize(void *element)
 	struct IO_Event_Selector_URing_Completion *completion = element;
 	IO_Event_List_initialize(&completion->list);
 	completion->list.type = &IO_Event_Selector_URing_Completion_Type;
+	completion->waiting = NULL;
+	completion->operation_pending = false;
+	completion->cancellation_pending = false;
 }
 
 void IO_Event_Selector_URing_Completion_free(void *element)
@@ -237,8 +306,8 @@ VALUE IO_Event_Selector_URing_allocate(VALUE self) {
 	
 	IO_Event_Selector_initialize(&selector->backend, self, Qnil);
 	selector->ring.ring_fd = -1;
+	selector->owner = 0;
 	
-	selector->blocked = 0;
 	selector->interrupt.descriptor = -1;
 	selector->wakeup_registered = 0;
 	
@@ -299,6 +368,8 @@ VALUE IO_Event_Selector_URing_initialize(VALUE self, VALUE loop) {
 		rb_syserr_fail(-result, "IO_Event_Selector_URing_initialize:io_uring_queue_init");
 	}
 	
+	selector->owner = getpid();
+	
 	rb_update_max_fd(selector->ring.ring_fd);
 	
 	// Interrupt for cross-thread wakeup: another thread calls signal(); the owner
@@ -339,6 +410,13 @@ VALUE IO_Event_Selector_URing_close(VALUE self) {
 	return Qnil;
 }
 
+VALUE IO_Event_Selector_URing_closed_p(VALUE self) {
+	struct IO_Event_Selector_URing *selector = NULL;
+	TypedData_Get_Struct(self, struct IO_Event_Selector_URing, &IO_Event_Selector_URing_Type, selector);
+	
+	return selector->ring.ring_fd < 0 || selector->owner != getpid() ? Qtrue : Qfalse;
+}
+
 VALUE IO_Event_Selector_URing_transfer(VALUE self)
 {
 	struct IO_Event_Selector_URing *selector = NULL;
@@ -480,37 +558,185 @@ struct io_uring_sqe * io_get_sqe(struct IO_Event_Selector_URing *selector) {
 	return sqe;
 }
 
+static
+void IO_Event_Selector_URing_Completion_cancel_async(struct IO_Event_Selector_URing *selector, struct IO_Event_Selector_URing_Completion *completion)
+{
+	if (completion->cancellation_pending) return;
+	
+	completion->cancellation_pending = true;
+	
+	struct io_uring_sqe *sqe = io_get_sqe(selector);
+	io_uring_prep_cancel(sqe, completion, 0);
+	io_uring_sqe_set_data64(sqe, IO_Event_Selector_URing_Completion_cancellation_data(completion));
+	io_uring_submit_pending(selector);
+}
+
+static
+VALUE IO_Event_Selector_URing_Waiting_wait_yield(VALUE _selector)
+{
+	struct IO_Event_Selector_URing *selector = (struct IO_Event_Selector_URing *)_selector;
+	
+	return IO_Event_Selector_loop_yield(&selector->backend);
+}
+
+static
+void IO_Event_Selector_URing_Waiting_cancel_and_wait(struct IO_Event_Selector_URing *selector, struct IO_Event_Selector_URing_Waiting *waiting)
+{
+	int state = 0;
+	VALUE error = Qnil;
+	
+	if (waiting->completion) {
+		IO_Event_Selector_URing_Completion_cancel_async(selector, waiting->completion);
+		
+		// The kernel may still be reading from or writing to the supplied buffer.
+		// Keep the C frame, buffer lock and completion record alive until the
+		// original operation CQE confirms that it can no longer access memory.
+		while (waiting->completion) {
+			int current_state = 0;
+			rb_protect(IO_Event_Selector_URing_Waiting_wait_yield, (VALUE)selector, &current_state);
+			
+			if (current_state && !state) {
+				state = current_state;
+				error = rb_errinfo();
+			}
+			
+			rb_set_errinfo(Qnil);
+		}
+	}
+	
+	IO_Event_Selector_URing_Waiting_cancel(waiting);
+	
+	if (state) {
+		rb_set_errinfo(error);
+		rb_jump_tag(state);
+	}
+}
+
 #pragma mark - Process.wait
 
+#ifdef IO_EVENT_SELECTOR_URING_USE_WAITID
+// Translate a Ruby/`waitpid`-style pid into the `waitid(2)` idtype and id, mirroring the semantics of `waitpid(2)`:
+//
+//   pid == -1  -> any child                                  (P_ALL)
+//   pid ==  0  -> any child in the caller's process group    (P_PGID, id 0; Linux >= 5.4)
+//   pid <  -1  -> any child in process group |pid|           (P_PGID)
+//   pid >   0  -> the specific child                         (P_PID)
+//
+static inline idtype_t process_waitid_type(pid_t pid, id_t *id) {
+	if (pid == -1) {
+		*id = 0;
+		return P_ALL;
+	} else if (pid == 0) {
+		*id = 0;
+		return P_PGID;
+	} else if (pid < -1) {
+		*id = (id_t)(-pid);
+		return P_PGID;
+	} else {
+		*id = (id_t)pid;
+		return P_PID;
+	}
+}
+#endif
+
 struct process_wait_arguments {
 	struct IO_Event_Selector_URing *selector;
 	struct IO_Event_Selector_URing_Waiting *waiting;
 	
 	pid_t pid;
 	int flags;
+	
+#ifdef IO_EVENT_SELECTOR_URING_USE_WAITID
+	siginfo_t siginfo;
+#else
 	int descriptor;
+#endif
 };
 
+#if defined(IO_EVENT_SELECTOR_URING_USE_WAITID) && defined(HAVE_RB_PROCESS_STATUS_FOR)
+static inline int process_wait_status_exited(int exit_status) {
+	return (exit_status & 0xff) << 8;
+}
+
+static inline int process_wait_status_signaled(int term_signal) {
+	return term_signal & 0x7f;
+}
+
+static inline int process_wait_status_from_siginfo(const siginfo_t *siginfo) {
+	switch (siginfo->si_code) {
+		case CLD_EXITED:
+			return process_wait_status_exited(siginfo->si_status);
+		case CLD_KILLED:
+			return process_wait_status_signaled(siginfo->si_status);
+		case CLD_DUMPED:
+			return process_wait_status_signaled(siginfo->si_status) | 0x80;
+		default:
+			return 0;
+	}
+}
+#endif
+
 static
 VALUE process_wait_transfer(VALUE _arguments) {
 	struct process_wait_arguments *arguments = (struct process_wait_arguments *)_arguments;
 	
+#ifdef IO_EVENT_SELECTOR_URING_USE_WAITID
+	IO_Event_Selector_loop_yield(&arguments->selector->backend);
+	
+	if (arguments->waiting->completion) {
+		// The kernel may still write to siginfo or reap the child. Keep the C frame alive until the original operation completes, then distinguish cancellation from process completion:
+		IO_Event_Selector_URing_Waiting_cancel_and_wait(arguments->selector, arguments->waiting);
+	}
+	
+	int32_t result = arguments->waiting->result;
+	if (result == -ECANCELED) {
+		return Qfalse;
+	}
+	
+	if (DEBUG) fprintf(stderr, "waitid result=%d pid=%d code=%d status=%d\n", result, arguments->siginfo.si_pid, arguments->siginfo.si_code, arguments->siginfo.si_status);
+	
+	if (result < 0) {
+		// The `waitid` failed (e.g. `ECHILD` when there are no children). Reproduce the failure as a `Process::Status` carrying the error, rather than raising, so callers like `Process.waitall` / `Process.detach` (which expect `waitpid` to report the error, not raise) behave correctly:
+#ifdef HAVE_RB_PROCESS_STATUS_FOR
+		return rb_process_status_for(-1, 0, -result);
+#else
+		return IO_Event_Selector_process_status_reap(arguments->pid, arguments->flags);
+#endif
+	}
+	
+#ifdef HAVE_RB_PROCESS_STATUS_FOR
+	// The `waitid` operation already reaped the child. Convert the `siginfo_t` result directly into the Ruby process status value:
+	return rb_process_status_for(arguments->siginfo.si_pid, process_wait_status_from_siginfo(&arguments->siginfo), 0);
+#else
+	// We waited with `WNOWAIT`, so the child has not been reaped yet. `si_pid` tells us exactly which child changed state (important when waiting for any child, e.g. pid -1). Reap it to obtain a correct `Process::Status`:
+	return IO_Event_Selector_process_status_reap(arguments->siginfo.si_pid, arguments->flags);
+#endif
+#else
 	IO_Event_Selector_loop_yield(&arguments->selector->backend);
 	
 	if (arguments->waiting->result) {
-		return IO_Event_Selector_process_status_wait(arguments->pid, arguments->flags);
+		return IO_Event_Selector_process_status_reap(arguments->pid, arguments->flags);
 	} else {
 		return Qfalse;
 	}
+#endif
 }
 
 static
 VALUE process_wait_ensure(VALUE _arguments) {
 	struct process_wait_arguments *arguments = (struct process_wait_arguments *)_arguments;
 	
-	close(arguments->descriptor);
+#ifdef IO_EVENT_SELECTOR_URING_USE_WAITID
+	// `waitid` may write to stack-backed siginfo while cancellation is pending:
+	IO_Event_Selector_URing_Waiting_cancel_and_wait(arguments->selector, arguments->waiting);
+#else
+	if (arguments->waiting->completion) {
+		IO_Event_Selector_URing_Completion_cancel_async(arguments->selector, arguments->waiting->completion);
+	}
 	
+	close(arguments->descriptor);
 	IO_Event_Selector_URing_Waiting_cancel(arguments->waiting);
+#endif
 	
 	return Qnil;
 }
@@ -522,11 +748,18 @@ VALUE IO_Event_Selector_URing_process_wait(VALUE self, VALUE fiber, VALUE _pid,
 	pid_t pid = NUM2PIDT(_pid);
 	int flags = NUM2INT(_flags);
 	
+#ifndef IO_EVENT_SELECTOR_URING_USE_WAITID
+	// `pidfd_open` can only refer to a specific process, so waiting for any child or a process group (pid <= 0) is delegated to the threaded fallback:
+	if (pid <= 0) {
+		return IO_Event_Selector_process_wait(pid, flags);
+	}
+	
 	int descriptor = pidfd_open(pid, 0);
 	if (descriptor < 0) {
 		rb_syserr_fail(errno, "IO_Event_Selector_URing_process_wait:pidfd_open");
 	}
 	rb_update_max_fd(descriptor);
+#endif
 	
 	struct IO_Event_Selector_URing_Waiting waiting = {
 		.fiber = fiber,
@@ -541,12 +774,30 @@ VALUE IO_Event_Selector_URing_process_wait(VALUE self, VALUE fiber, VALUE _pid,
 		.waiting = &waiting,
 		.pid = pid,
 		.flags = flags,
+#ifdef IO_EVENT_SELECTOR_URING_USE_WAITID
+		.siginfo = {0},
+#else
 		.descriptor = descriptor,
+#endif
 	};
 	
-	if (DEBUG) fprintf(stderr, "IO_Event_Selector_URing_process_wait:io_uring_prep_poll_add(%p)\n", (void*)fiber);
 	struct io_uring_sqe *sqe = io_get_sqe(selector);
+	
+#ifdef IO_EVENT_SELECTOR_URING_USE_WAITID
+	id_t id;
+	idtype_t idtype = process_waitid_type(pid, &id);
+	if (DEBUG) fprintf(stderr, "IO_Event_Selector_URing_process_wait:io_uring_prep_waitid(fiber=%p, idtype=%d, id=%d, flags=%d)\n", (void*)fiber, idtype, (int)id, flags);
+#ifdef HAVE_RB_PROCESS_STATUS_FOR
+	// Reap the child directly; the completion contains enough information to construct the Ruby process status value:
+	io_uring_prep_waitid(sqe, idtype, id, &process_wait_arguments.siginfo, WEXITED, 0);
+#else
+	// `WNOWAIT` leaves the child in a waitable state so we can reap it with `rb_process_status_wait` afterwards and build a correct `Process::Status`:
+	io_uring_prep_waitid(sqe, idtype, id, &process_wait_arguments.siginfo, WEXITED | WNOWAIT, 0);
+#endif
+#else
+	if (DEBUG) fprintf(stderr, "IO_Event_Selector_URing_process_wait:io_uring_prep_poll_add(%p)\n", (void*)fiber);
 	io_uring_prep_poll_add(sqe, descriptor, POLLIN|POLLHUP|POLLERR);
+#endif
 	io_uring_sqe_set_data(sqe, completion);
 	io_uring_submit_pending(selector);
 	
@@ -591,13 +842,8 @@ static
 VALUE io_wait_ensure(VALUE _arguments) {
 	struct io_wait_arguments *arguments = (struct io_wait_arguments *)_arguments;
 	
-	// If the operation is still in progress, cancel it:
 	if (arguments->waiting->completion) {
-		if (DEBUG) fprintf(stderr, "io_wait_ensure:io_uring_prep_cancel(waiting=%p, completion=%p)\n", (void*)arguments->waiting, (void*)arguments->waiting->completion);
-		struct io_uring_sqe *sqe = io_get_sqe(arguments->selector);
-		io_uring_prep_cancel(sqe, (void*)arguments->waiting->completion, 0);
-		io_uring_sqe_set_data(sqe, NULL);
-		io_uring_submit_now(arguments->selector);
+		IO_Event_Selector_URing_Completion_cancel_async(arguments->selector, arguments->waiting->completion);
 	}
 	
 	IO_Event_Selector_URing_Waiting_cancel(arguments->waiting);
@@ -701,7 +947,7 @@ io_read_submit(VALUE _arguments)
 	struct io_uring_sqe *sqe = io_get_sqe(selector);
 	io_uring_prep_read(sqe, arguments->descriptor, arguments->buffer, arguments->length, arguments->offset);
 	io_uring_sqe_set_data(sqe, arguments->waiting->completion);
-	io_uring_submit_now(selector);
+	io_uring_submit_pending(selector);
 	
 	IO_Event_Selector_loop_yield(&selector->backend);
 	
@@ -714,16 +960,7 @@ io_read_ensure(VALUE _arguments)
 	struct io_read_arguments *arguments = (struct io_read_arguments *)_arguments;
 	struct IO_Event_Selector_URing *selector = arguments->selector;
 	
-	// If the operation is still in progress, cancel it:
-	if (arguments->waiting->completion) {
-		if (DEBUG) fprintf(stderr, "io_read_ensure:io_uring_prep_cancel(waiting=%p, completion=%p)\n", (void*)arguments->waiting, (void*)arguments->waiting->completion);
-		struct io_uring_sqe *sqe = io_get_sqe(selector);
-		io_uring_prep_cancel(sqe, (void*)arguments->waiting->completion, 0);
-		io_uring_sqe_set_data(sqe, NULL);
-		io_uring_submit_now(selector);
-	}
-	
-	IO_Event_Selector_URing_Waiting_cancel(arguments->waiting);
+	IO_Event_Selector_URing_Waiting_cancel_and_wait(selector, arguments->waiting);
 	
 	return Qnil;
 }
@@ -753,26 +990,97 @@ io_read(struct IO_Event_Selector_URing *selector, VALUE fiber, int descriptor, c
 	);
 }
 
-VALUE IO_Event_Selector_URing_io_read(VALUE self, VALUE fiber, VALUE io, VALUE buffer, VALUE _length, VALUE _offset) {
+#if RUBY_FIBER_SCHEDULER_VERSION >= 4
+struct io_read_locked_arguments {
+	struct IO_Event_Selector_URing *selector;
+	VALUE fiber;
+	VALUE io;
+	off_t from;
+	size_t offset;
+	size_t length;
+	bool positional;
+};
+
+static VALUE
+io_read_locked(void *base, size_t size, VALUE _arguments)
+{
+	struct io_read_locked_arguments *arguments = (struct io_read_locked_arguments *)_arguments;
+	
+	if (!IO_Event_Selector_valid_buffer_range(size, arguments->offset, arguments->length)) {
+		return rb_fiber_scheduler_io_result(-1, EINVAL);
+	} else if (arguments->length == 0) {
+		return rb_fiber_scheduler_io_result(0, 0);
+	}
+	
+	int descriptor = IO_Event_Selector_io_descriptor(arguments->io);
+	char *buffer = (char*)base + arguments->offset;
+	
+	// Avoid the submission and suspension overhead when the operation can
+	// complete immediately. The descriptor must be non-blocking for this
+	// optimistic syscall; if it would block, restore its original mode and
+	// submit the operation to io_uring instead.
+	int flags = IO_Event_Selector_nonblock_set(descriptor);
+	ssize_t result;
+	
+	if (arguments->positional) {
+		result = pread(descriptor, buffer, arguments->length, arguments->from);
+	} else {
+		result = read(descriptor, buffer, arguments->length);
+	}
+	
+	int error = errno;
+	IO_Event_Selector_nonblock_restore(descriptor, flags);
+	
+	if (result >= 0) {
+		return rb_fiber_scheduler_io_result(result, 0);
+	} else if (!IO_Event_try_again(error)) {
+		return rb_fiber_scheduler_io_result(-1, error);
+	}
+	
+	off_t from = arguments->positional ? arguments->from : io_seekable(descriptor);
+	
+	int completion = io_read(arguments->selector, arguments->fiber, descriptor, buffer, arguments->length, from);
+	if (completion < 0) {
+		return rb_fiber_scheduler_io_result(-1, -completion);
+	}
+	
+	return rb_fiber_scheduler_io_result(completion, 0);
+}
+#endif
+
+VALUE IO_Event_Selector_URing_io_read(VALUE self, VALUE fiber, VALUE io, VALUE buffer, VALUE _first, VALUE _second) {
 	struct IO_Event_Selector_URing *selector = NULL;
 	TypedData_Get_Struct(self, struct IO_Event_Selector_URing, &IO_Event_Selector_URing_Type, selector);
 	
-	int descriptor = IO_Event_Selector_io_descriptor(io);
+#if RUBY_FIBER_SCHEDULER_VERSION >= 4
+	struct io_read_locked_arguments arguments = {
+		.selector = selector,
+		.fiber = fiber,
+		.io = io,
+		.offset = NUM2SIZET(_first),
+		.length = NUM2SIZET(_second),
+		.positional = false,
+	};
 	
+	return rb_io_buffer_locked_for_writing(buffer, io_read_locked, (VALUE)&arguments);
+#else
 	void *base;
 	size_t size;
 	rb_io_buffer_get_bytes_for_writing(buffer, &base, &size);
 	
-	size_t length = NUM2SIZET(_length);
-	size_t offset = NUM2SIZET(_offset);
+	size_t length = NUM2SIZET(_first);
+	size_t offset = NUM2SIZET(_second);
 	size_t total = 0;
-	off_t from = io_seekable(descriptor);
 	
 	// Ensure offset is within the bounds of the buffer to avoid size_t underflow and out-of-bounds pointer arithmetic on (char *)base + offset.
 	if (offset > size) {
 		return rb_fiber_scheduler_io_result(-1, EINVAL);
+	} else if (offset == size) {
+		return rb_fiber_scheduler_io_result(0, 0);
 	}
 	
+	int descriptor = IO_Event_Selector_io_descriptor(io);
+	off_t from = io_seekable(descriptor);
 	size_t maximum_size = size - offset;
 	
 	// Are we performing a non-blocking read?
@@ -808,8 +1116,10 @@ VALUE IO_Event_Selector_URing_io_read(VALUE self, VALUE fiber, VALUE io, VALUE b
 	}
 	
 	return rb_fiber_scheduler_io_result(total, 0);
+#endif
 }
 
+#if RUBY_FIBER_SCHEDULER_VERSION < 4
 static VALUE IO_Event_Selector_URing_io_read_compatible(int argc, VALUE *argv, VALUE self)
 {
 	rb_check_arity(argc, 4, 5);
@@ -822,27 +1132,41 @@ static VALUE IO_Event_Selector_URing_io_read_compatible(int argc, VALUE *argv, V
 	
 	return IO_Event_Selector_URing_io_read(self, argv[0], argv[1], argv[2], argv[3], _offset);
 }
+#endif
 
-VALUE IO_Event_Selector_URing_io_pread(VALUE self, VALUE fiber, VALUE io, VALUE buffer, VALUE _from, VALUE _length, VALUE _offset) {
+VALUE IO_Event_Selector_URing_io_pread(VALUE self, VALUE fiber, VALUE io, VALUE buffer, VALUE _from, VALUE _first, VALUE _second) {
 	struct IO_Event_Selector_URing *selector = NULL;
 	TypedData_Get_Struct(self, struct IO_Event_Selector_URing, &IO_Event_Selector_URing_Type, selector);
 	
-	int descriptor = IO_Event_Selector_io_descriptor(io);
+#if RUBY_FIBER_SCHEDULER_VERSION >= 4
+	struct io_read_locked_arguments arguments = {
+		.selector = selector,
+		.fiber = fiber,
+		.io = io,
+		.from = NUM2OFFT(_from),
+		.offset = NUM2SIZET(_first),
+		.length = NUM2SIZET(_second),
+		.positional = true,
+	};
 	
+	return rb_io_buffer_locked_for_writing(buffer, io_read_locked, (VALUE)&arguments);
+#else
 	void *base;
 	size_t size;
 	rb_io_buffer_get_bytes_for_writing(buffer, &base, &size);
 	
-	size_t length = NUM2SIZET(_length);
-	size_t offset = NUM2SIZET(_offset);
+	size_t length = NUM2SIZET(_first);
+	size_t offset = NUM2SIZET(_second);
 	size_t total = 0;
-	off_t from = NUM2OFFT(_from);
 	
 	// Ensure offset is within the bounds of the buffer to avoid size_t underflow and out-of-bounds pointer arithmetic on (char *)base + offset.
 	if (offset > size) {
 		return rb_fiber_scheduler_io_result(-1, EINVAL);
+	} else if (offset == size) {
+		return rb_fiber_scheduler_io_result(0, 0);
 	}
-	
+	off_t from = NUM2OFFT(_from);
+	int descriptor = IO_Event_Selector_io_descriptor(io);
 	size_t maximum_size = size - offset;
 	while (maximum_size) {
 		int result = io_read(selector, fiber, descriptor, (char*)base+offset, maximum_size, from);
@@ -865,6 +1189,7 @@ VALUE IO_Event_Selector_URing_io_pread(VALUE self, VALUE fiber, VALUE io, VALUE
 	}
 	
 	return rb_fiber_scheduler_io_result(total, 0);
+#endif
 }
 
 #pragma mark - IO#write
@@ -902,16 +1227,7 @@ io_write_ensure(VALUE _argument)
 	struct io_write_arguments *arguments = (struct io_write_arguments*)_argument;
 	struct IO_Event_Selector_URing *selector = arguments->selector;
 	
-	// If the operation is still in progress, cancel it:
-	if (arguments->waiting->completion) {
-		if (DEBUG) fprintf(stderr, "io_write_ensure:io_uring_prep_cancel(waiting=%p, completion=%p)\n", (void*)arguments->waiting, (void*)arguments->waiting->completion);
-		struct io_uring_sqe *sqe = io_get_sqe(selector);
-		io_uring_prep_cancel(sqe, (void*)arguments->waiting->completion, 0);
-		io_uring_sqe_set_data(sqe, NULL);
-		io_uring_submit_now(selector);
-	}
-	
-	IO_Event_Selector_URing_Waiting_cancel(arguments->waiting);
+	IO_Event_Selector_URing_Waiting_cancel_and_wait(selector, arguments->waiting);
 	
 	return Qnil;
 }
@@ -941,20 +1257,87 @@ io_write(struct IO_Event_Selector_URing *selector, VALUE fiber, int descriptor,
 	);
 }
 
-VALUE IO_Event_Selector_URing_io_write(VALUE self, VALUE fiber, VALUE io, VALUE buffer, VALUE _length, VALUE _offset) {
+#if RUBY_FIBER_SCHEDULER_VERSION >= 4
+struct io_write_locked_arguments {
+	struct IO_Event_Selector_URing *selector;
+	VALUE fiber;
+	VALUE io;
+	off_t from;
+	size_t offset;
+	size_t length;
+	bool positional;
+};
+
+static VALUE
+io_write_locked(const void *base, size_t size, VALUE _arguments)
+{
+	struct io_write_locked_arguments *arguments = (struct io_write_locked_arguments *)_arguments;
+	
+	if (!IO_Event_Selector_valid_buffer_range(size, arguments->offset, arguments->length)) {
+		return rb_fiber_scheduler_io_result(-1, EINVAL);
+	} else if (arguments->length == 0) {
+		return rb_fiber_scheduler_io_result(0, 0);
+	}
+	
+	int descriptor = IO_Event_Selector_io_descriptor(arguments->io);
+	const char *buffer = (const char*)base + arguments->offset;
+	
+	// Avoid the submission and suspension overhead when the operation can
+	// complete immediately. The descriptor must be non-blocking for this
+	// optimistic syscall; if it would block, restore its original mode and
+	// submit the operation to io_uring instead.
+	int flags = IO_Event_Selector_nonblock_set(descriptor);
+	ssize_t result;
+	
+	if (arguments->positional) {
+		result = pwrite(descriptor, buffer, arguments->length, arguments->from);
+	} else {
+		result = write(descriptor, buffer, arguments->length);
+	}
+	
+	int error = errno;
+	IO_Event_Selector_nonblock_restore(descriptor, flags);
+	
+	if (result >= 0) {
+		return rb_fiber_scheduler_io_result(result, 0);
+	} else if (!IO_Event_try_again(error)) {
+		return rb_fiber_scheduler_io_result(-1, error);
+	}
+	
+	off_t from = arguments->positional ? arguments->from : io_seekable(descriptor);
+	
+	int completion = io_write(arguments->selector, arguments->fiber, descriptor, (char*)buffer, arguments->length, from);
+	if (completion < 0) {
+		return rb_fiber_scheduler_io_result(-1, -completion);
+	}
+	
+	return rb_fiber_scheduler_io_result(completion, 0);
+}
+#endif
+
+VALUE IO_Event_Selector_URing_io_write(VALUE self, VALUE fiber, VALUE io, VALUE buffer, VALUE _first, VALUE _second) {
 	struct IO_Event_Selector_URing *selector = NULL;
 	TypedData_Get_Struct(self, struct IO_Event_Selector_URing, &IO_Event_Selector_URing_Type, selector);
 	
-	int descriptor = IO_Event_Selector_io_descriptor(io);
+#if RUBY_FIBER_SCHEDULER_VERSION >= 4
+	struct io_write_locked_arguments arguments = {
+		.selector = selector,
+		.fiber = fiber,
+		.io = io,
+		.offset = NUM2SIZET(_first),
+		.length = NUM2SIZET(_second),
+		.positional = false,
+	};
 	
+	return rb_io_buffer_locked_for_reading(buffer, io_write_locked, (VALUE)&arguments);
+#else
 	const void *base;
 	size_t size;
 	rb_io_buffer_get_bytes_for_reading(buffer, &base, &size);
 	
-	size_t length = NUM2SIZET(_length);
-	size_t offset = NUM2SIZET(_offset);
+	size_t length = NUM2SIZET(_first);
+	size_t offset = NUM2SIZET(_second);
 	size_t total = 0;
-	off_t from = io_seekable(descriptor);
 	
 	if (length > size) {
 		rb_raise(rb_eRuntimeError, "Length exceeds size of buffer!");
@@ -963,8 +1346,12 @@ VALUE IO_Event_Selector_URing_io_write(VALUE self, VALUE fiber, VALUE io, VALUE
 	// Ensure offset is within the bounds of the buffer to avoid size_t underflow and out-of-bounds pointer arithmetic on (char *)base + offset.
 	if (offset > size) {
 		return rb_fiber_scheduler_io_result(-1, EINVAL);
+	} else if (offset == size) {
+		return rb_fiber_scheduler_io_result(0, 0);
 	}
-
+	
+	int descriptor = IO_Event_Selector_io_descriptor(io);
+	off_t from = io_seekable(descriptor);
 	size_t maximum_size = size - offset;
 	while (maximum_size) {
 		int result = io_write(selector, fiber, descriptor, (char*)base+offset, maximum_size, from);
@@ -986,8 +1373,10 @@ VALUE IO_Event_Selector_URing_io_write(VALUE self, VALUE fiber, VALUE io, VALUE
 	}
 	
 	return rb_fiber_scheduler_io_result(total, 0);
+#endif
 }
 
+#if RUBY_FIBER_SCHEDULER_VERSION < 4
 static VALUE IO_Event_Selector_URing_io_write_compatible(int argc, VALUE *argv, VALUE self)
 {
 	rb_check_arity(argc, 4, 5);
@@ -1000,21 +1389,32 @@ static VALUE IO_Event_Selector_URing_io_write_compatible(int argc, VALUE *argv,
 	
 	return IO_Event_Selector_URing_io_write(self, argv[0], argv[1], argv[2], argv[3], _offset);
 }
+#endif
 
-VALUE IO_Event_Selector_URing_io_pwrite(VALUE self, VALUE fiber, VALUE io, VALUE buffer, VALUE _from, VALUE _length, VALUE _offset) {
+VALUE IO_Event_Selector_URing_io_pwrite(VALUE self, VALUE fiber, VALUE io, VALUE buffer, VALUE _from, VALUE _first, VALUE _second) {
 	struct IO_Event_Selector_URing *selector = NULL;
 	TypedData_Get_Struct(self, struct IO_Event_Selector_URing, &IO_Event_Selector_URing_Type, selector);
 	
-	int descriptor = IO_Event_Selector_io_descriptor(io);
+#if RUBY_FIBER_SCHEDULER_VERSION >= 4
+	struct io_write_locked_arguments arguments = {
+		.selector = selector,
+		.fiber = fiber,
+		.io = io,
+		.from = NUM2OFFT(_from),
+		.offset = NUM2SIZET(_first),
+		.length = NUM2SIZET(_second),
+		.positional = true,
+	};
 	
+	return rb_io_buffer_locked_for_reading(buffer, io_write_locked, (VALUE)&arguments);
+#else
 	const void *base;
 	size_t size;
 	rb_io_buffer_get_bytes_for_reading(buffer, &base, &size);
 	
-	size_t length = NUM2SIZET(_length);
-	size_t offset = NUM2SIZET(_offset);
+	size_t length = NUM2SIZET(_first);
+	size_t offset = NUM2SIZET(_second);
 	size_t total = 0;
-	off_t from = NUM2OFFT(_from);
 	
 	if (length > size) {
 		rb_raise(rb_eRuntimeError, "Length exceeds size of buffer!");
@@ -1023,8 +1423,11 @@ VALUE IO_Event_Selector_URing_io_pwrite(VALUE self, VALUE fiber, VALUE io, VALUE
 	// Ensure offset is within the bounds of the buffer to avoid size_t underflow and out-of-bounds pointer arithmetic on (char *)base + offset.
 	if (offset > size) {
 		return rb_fiber_scheduler_io_result(-1, EINVAL);
+	} else if (offset == size) {
+		return rb_fiber_scheduler_io_result(0, 0);
 	}
-
+	off_t from = NUM2OFFT(_from);
+	int descriptor = IO_Event_Selector_io_descriptor(io);
 	size_t maximum_size = size - offset;
 	while (maximum_size) {
 		int result = io_write(selector, fiber, descriptor, (char*)base+offset, maximum_size, from);
@@ -1047,6 +1450,7 @@ VALUE IO_Event_Selector_URing_io_pwrite(VALUE self, VALUE fiber, VALUE io, VALUE
 	}
 	
 	return rb_fiber_scheduler_io_result(total, 0);
+#endif
 }
 
 #endif
@@ -1104,9 +1508,14 @@ struct __kernel_timespec * make_timeout(VALUE duration, struct __kernel_timespec
 	return storage;
 }
 
+// Return true when it is safe and useful to enter the blocking selector wait.
 static
-int timeout_nonblocking(struct __kernel_timespec *timespec) {
-	return timespec && timespec->tv_sec == 0 && timespec->tv_nsec == 0;
+int select_blocking_allowed(struct __kernel_timespec *timespec) {
+	// A `0` timeout is already a poll. The selector has already performed the immediate poll previously, so there is no useful blocking wait to enter here.
+	if (timespec && timespec->tv_sec == 0 && timespec->tv_nsec == 0) {
+		return 0;
+	}
+	return 1;
 }
 
 struct select_arguments {
@@ -1145,22 +1554,21 @@ int select_internal_without_gvl(struct select_arguments *arguments) {
 	
 	io_uring_submit_flush(selector);
 	
-	selector->blocked = 1;
-	rb_thread_call_without_gvl(select_internal, (void *)arguments, RUBY_UBF_IO, 0);
-	selector->blocked = 0;
+	arguments->result = -EINTR;
+	IO_Event_Selector_blocking_operation(&selector->backend, select_internal, (void *)arguments, RUBY_UBF_IO, 0);
 	
 	if (arguments->result == -ETIME) {
-		arguments->result = 0;
+		return 0;
 	} else if (arguments->result == -EINTR) {
-		arguments->result = 0;
+		return 0;
 	} else if (arguments->result < 0) {
 		rb_syserr_fail(-arguments->result, "select_internal_without_gvl:io_uring_wait_cqe_timeout");
 	} else {
 		// At least 1 event is waiting:
-		arguments->result = 1;
+		return 1;
 	}
 	
-	return arguments->result;
+	return 0;
 }
 
 static inline
@@ -1194,6 +1602,13 @@ unsigned select_process_completions(struct IO_Event_Selector_URing *selector) {
 			continue;
 		}
 		
+		if (cqe->user_data & IO_EVENT_SELECTOR_URING_CANCELLATION_TAG) {
+			struct IO_Event_Selector_URing_Completion *completion = IO_Event_Selector_URing_Completion_from_cancellation_data(cqe->user_data);
+			IO_Event_Selector_URing_Completion_cancellation_complete(selector, completion);
+			io_uring_cq_advance(ring, 1);
+			continue;
+		}
+		
 		struct IO_Event_Selector_URing_Completion *completion = (void*)cqe->user_data;
 		struct IO_Event_Selector_URing_Waiting *waiting = completion->waiting;
 		
@@ -1208,13 +1623,11 @@ unsigned select_process_completions(struct IO_Event_Selector_URing *selector) {
 		
 		VALUE fiber = 0;
 		if (waiting && waiting->fiber) {
-			assert(waiting->result != -ECANCELED);
-			
 			fiber = waiting->fiber;
 		}
 
 		// This marks the waiting operation as "complete":
-		IO_Event_Selector_URing_Completion_release(selector, completion);
+		IO_Event_Selector_URing_Completion_complete(selector, completion);
 		
 		if (fiber) {
 			IO_Event_Selector_loop_resume(&selector->backend, fiber, 0, NULL);
@@ -1257,28 +1670,29 @@ VALUE IO_Event_Selector_URing_select(VALUE self, VALUE duration) {
 	
 	int ready = IO_Event_Selector_ready_flush(&selector->backend);
 	
-	int result = select_process_completions(selector);
+	int completed = select_process_completions(selector);
 	
 	// If we:
 	// 1. Didn't process any ready fibers, and
 	// 2. Didn't process any events from non-blocking select (above), and
 	// 3. There are no items in the ready list,
 	// then we can perform a blocking select.
-	if (!ready && !result && !selector->backend.ready) {
+	if (!ready && !completed && !selector->backend.ready) {
 		// We might need to wait for events:
 		struct select_arguments arguments = {
 			.selector = selector,
+			.result = 0,
 			.timeout = NULL,
 		};
 		
 		arguments.timeout = make_timeout(duration, &arguments.storage);
 		
-		if (!selector->backend.ready && !timeout_nonblocking(arguments.timeout)) {
+		if (!selector->backend.ready && select_blocking_allowed(arguments.timeout)) {
 			struct timespec start_time;
 			IO_Event_Time_current(&start_time);
 			
 			// This is a blocking operation, we wait for events:
-			result = select_internal_without_gvl(&arguments);
+			int result = select_internal_without_gvl(&arguments);
 			
 			struct timespec end_time;
 			IO_Event_Time_current(&end_time);
@@ -1286,12 +1700,12 @@ VALUE IO_Event_Selector_URing_select(VALUE self, VALUE duration) {
 			
 			// After waiting/flushing the SQ, check if there are any completions:
 			if (result > 0) {
-				result = select_process_completions(selector);
+				completed = select_process_completions(selector);
 			}
 		}
 	}
 	
-	return RB_INT2NUM(result);
+	return RB_INT2NUM(completed);
 }
 
 VALUE IO_Event_Selector_URing_wakeup(VALUE self) {
@@ -1300,7 +1714,7 @@ VALUE IO_Event_Selector_URing_wakeup(VALUE self) {
 	
 	// Wake the selector by signalling the interrupt. This is safe from any thread
 	// and never touches the ring's SQ, which is required for IORING_SETUP_SINGLE_ISSUER.
-	if (selector->blocked) {
+	if (selector->backend.blocked) {
 		IO_Event_Interrupt_signal(&selector->interrupt);
 		return Qtrue;
 	}
@@ -1370,12 +1784,18 @@ void Init_IO_Event_Selector_URing(VALUE IO_Event_Selector) {
 	rb_define_method(IO_Event_Selector_URing, "select", IO_Event_Selector_URing_select, 1);
 	rb_define_method(IO_Event_Selector_URing, "wakeup", IO_Event_Selector_URing_wakeup, 0);
 	rb_define_method(IO_Event_Selector_URing, "close", IO_Event_Selector_URing_close, 0);
+	rb_define_method(IO_Event_Selector_URing, "closed?", IO_Event_Selector_URing_closed_p, 0);
 	
 	rb_define_method(IO_Event_Selector_URing, "io_wait", IO_Event_Selector_URing_io_wait, 3);
 	
 #ifdef HAVE_RUBY_IO_BUFFER_H
+#if RUBY_FIBER_SCHEDULER_VERSION >= 4
+	rb_define_method(IO_Event_Selector_URing, "io_read", IO_Event_Selector_URing_io_read, 5);
+	rb_define_method(IO_Event_Selector_URing, "io_write", IO_Event_Selector_URing_io_write, 5);
+#else
 	rb_define_method(IO_Event_Selector_URing, "io_read", IO_Event_Selector_URing_io_read_compatible, -1);
 	rb_define_method(IO_Event_Selector_URing, "io_write", IO_Event_Selector_URing_io_write_compatible, -1);
+#endif
 	rb_define_method(IO_Event_Selector_URing, "io_pread", IO_Event_Selector_URing_io_pread, 6);
 	rb_define_method(IO_Event_Selector_URing, "io_pwrite", IO_Event_Selector_URing_io_pwrite, 6);
 #endif