io-event 1.19.4 → 1.20.0

data/ext/io/event/selector/uring.c

@@ -81,8 +81,26 @@ struct IO_Event_Selector_URing_Completion
 	struct IO_Event_List list;
 	
 	struct IO_Event_Selector_URing_Waiting *waiting;
+	bool cancellation_pending;
 };
 
+// Cancellation completions use the low bit to distinguish them from the
+// completion of the operation being cancelled. Completion records are aligned,
+// so the low bit is otherwise always clear.
+#define IO_EVENT_SELECTOR_URING_CANCELLATION_TAG ((uint64_t)1)
+
+static inline uint64_t
+IO_Event_Selector_URing_Completion_cancellation_data(struct IO_Event_Selector_URing_Completion *completion)
+{
+	return ((uint64_t)(uintptr_t)completion) | IO_EVENT_SELECTOR_URING_CANCELLATION_TAG;
+}
+
+static inline struct IO_Event_Selector_URing_Completion *
+IO_Event_Selector_URing_Completion_from_cancellation_data(uint64_t data)
+{
+	return (struct IO_Event_Selector_URing_Completion *)(uintptr_t)(data & ~IO_EVENT_SELECTOR_URING_CANCELLATION_TAG);
+}
+
 static
 void IO_Event_Selector_URing_Completion_mark(void *_completion)
 {
@@ -188,6 +206,9 @@ struct IO_Event_Selector_URing_Completion * IO_Event_Selector_URing_Completion_a
 	
 	if (DEBUG_COMPLETION) fprintf(stderr, "IO_Event_Selector_URing_Completion_acquire(%p, limit=%ld)\n", (void*)completion, selector->completions.limit);
 	
+	assert(completion->waiting == NULL);
+	assert(!completion->cancellation_pending);
+	
 	waiting->completion = completion;
 	completion->waiting = waiting;
 	
@@ -206,14 +227,41 @@ void IO_Event_Selector_URing_Completion_cancel(struct IO_Event_Selector_URing_Co
 }
 
 inline static
-void IO_Event_Selector_URing_Completion_release(struct IO_Event_Selector_URing *selector, struct IO_Event_Selector_URing_Completion *completion)
+void IO_Event_Selector_URing_Completion_recycle(struct IO_Event_Selector_URing *selector, struct IO_Event_Selector_URing_Completion *completion)
 {
-	if (DEBUG_COMPLETION) fprintf(stderr, "IO_Event_Selector_URing_Completion_release(%p)\n", (void*)completion);
+	assert(completion->waiting == NULL);
+	assert(!completion->cancellation_pending);
 	
-	IO_Event_Selector_URing_Completion_cancel(completion);
 	IO_Event_List_prepend(&selector->free_list, &completion->list);
 }
 
+inline static
+void IO_Event_Selector_URing_Completion_complete(struct IO_Event_Selector_URing *selector, struct IO_Event_Selector_URing_Completion *completion)
+{
+	if (DEBUG_COMPLETION) fprintf(stderr, "IO_Event_Selector_URing_Completion_complete(%p)\n", (void*)completion);
+	
+	IO_Event_Selector_URing_Completion_cancel(completion);
+	
+	// A cancellation SQE still refers to this completion record. Keep it out of
+	// the free list until that CQE has also been observed, avoiding ABA reuse.
+	if (!completion->cancellation_pending) {
+		IO_Event_Selector_URing_Completion_recycle(selector, completion);
+	}
+}
+
+inline static
+void IO_Event_Selector_URing_Completion_cancellation_complete(struct IO_Event_Selector_URing *selector, struct IO_Event_Selector_URing_Completion *completion)
+{
+	if (DEBUG_COMPLETION) fprintf(stderr, "IO_Event_Selector_URing_Completion_cancellation_complete(%p)\n", (void*)completion);
+	
+	assert(completion->cancellation_pending);
+	completion->cancellation_pending = false;
+	
+	if (completion->waiting == NULL) {
+		IO_Event_Selector_URing_Completion_recycle(selector, completion);
+	}
+}
+
 inline static
 void IO_Event_Selector_URing_Waiting_cancel(struct IO_Event_Selector_URing_Waiting *waiting)
 {
@@ -234,6 +282,8 @@ void IO_Event_Selector_URing_Completion_initialize(void *element)
 	struct IO_Event_Selector_URing_Completion *completion = element;
 	IO_Event_List_initialize(&completion->list);
 	completion->list.type = &IO_Event_Selector_URing_Completion_Type;
+	completion->waiting = NULL;
+	completion->cancellation_pending = false;
 }
 
 void IO_Event_Selector_URing_Completion_free(void *element)
@@ -500,6 +550,36 @@ struct io_uring_sqe * io_get_sqe(struct IO_Event_Selector_URing *selector) {
 	return sqe;
 }
 
+static
+void IO_Event_Selector_URing_Completion_cancel_async(struct IO_Event_Selector_URing *selector, struct IO_Event_Selector_URing_Completion *completion)
+{
+	if (completion->cancellation_pending) return;
+	
+	completion->cancellation_pending = true;
+	
+	struct io_uring_sqe *sqe = io_get_sqe(selector);
+	io_uring_prep_cancel(sqe, completion, 0);
+	io_uring_sqe_set_data64(sqe, IO_Event_Selector_URing_Completion_cancellation_data(completion));
+	io_uring_submit_pending(selector);
+}
+
+static
+void IO_Event_Selector_URing_Waiting_cancel_and_wait(struct IO_Event_Selector_URing *selector, struct IO_Event_Selector_URing_Waiting *waiting)
+{
+	if (waiting->completion) {
+		IO_Event_Selector_URing_Completion_cancel_async(selector, waiting->completion);
+		
+		// The kernel may still be reading from or writing to the supplied buffer.
+		// Keep the C frame, buffer lock and completion record alive until the
+		// original operation CQE confirms that it can no longer access memory.
+		while (waiting->completion) {
+			IO_Event_Selector_loop_yield(&selector->backend);
+		}
+	}
+	
+	IO_Event_Selector_URing_Waiting_cancel(waiting);
+}
+
 #pragma mark - Process.wait
 
 #ifdef IO_EVENT_SELECTOR_URING_USE_WAITID
@@ -604,6 +684,10 @@ static
 VALUE process_wait_ensure(VALUE _arguments) {
 	struct process_wait_arguments *arguments = (struct process_wait_arguments *)_arguments;
 	
+	if (arguments->waiting->completion) {
+		IO_Event_Selector_URing_Completion_cancel_async(arguments->selector, arguments->waiting->completion);
+	}
+	
 #ifndef IO_EVENT_SELECTOR_URING_USE_WAITID
 	close(arguments->descriptor);
 #endif
@@ -714,13 +798,8 @@ static
 VALUE io_wait_ensure(VALUE _arguments) {
 	struct io_wait_arguments *arguments = (struct io_wait_arguments *)_arguments;
 	
-	// If the operation is still in progress, cancel it:
 	if (arguments->waiting->completion) {
-		if (DEBUG) fprintf(stderr, "io_wait_ensure:io_uring_prep_cancel(waiting=%p, completion=%p)\n", (void*)arguments->waiting, (void*)arguments->waiting->completion);
-		struct io_uring_sqe *sqe = io_get_sqe(arguments->selector);
-		io_uring_prep_cancel(sqe, (void*)arguments->waiting->completion, 0);
-		io_uring_sqe_set_data(sqe, NULL);
-		io_uring_submit_now(arguments->selector);
+		IO_Event_Selector_URing_Completion_cancel_async(arguments->selector, arguments->waiting->completion);
 	}
 	
 	IO_Event_Selector_URing_Waiting_cancel(arguments->waiting);
@@ -824,7 +903,7 @@ io_read_submit(VALUE _arguments)
 	struct io_uring_sqe *sqe = io_get_sqe(selector);
 	io_uring_prep_read(sqe, arguments->descriptor, arguments->buffer, arguments->length, arguments->offset);
 	io_uring_sqe_set_data(sqe, arguments->waiting->completion);
-	io_uring_submit_now(selector);
+	io_uring_submit_pending(selector);
 	
 	IO_Event_Selector_loop_yield(&selector->backend);
 	
@@ -837,16 +916,7 @@ io_read_ensure(VALUE _arguments)
 	struct io_read_arguments *arguments = (struct io_read_arguments *)_arguments;
 	struct IO_Event_Selector_URing *selector = arguments->selector;
 	
-	// If the operation is still in progress, cancel it:
-	if (arguments->waiting->completion) {
-		if (DEBUG) fprintf(stderr, "io_read_ensure:io_uring_prep_cancel(waiting=%p, completion=%p)\n", (void*)arguments->waiting, (void*)arguments->waiting->completion);
-		struct io_uring_sqe *sqe = io_get_sqe(selector);
-		io_uring_prep_cancel(sqe, (void*)arguments->waiting->completion, 0);
-		io_uring_sqe_set_data(sqe, NULL);
-		io_uring_submit_now(selector);
-	}
-	
-	IO_Event_Selector_URing_Waiting_cancel(arguments->waiting);
+	IO_Event_Selector_URing_Waiting_cancel_and_wait(selector, arguments->waiting);
 	
 	return Qnil;
 }
@@ -876,16 +946,86 @@ io_read(struct IO_Event_Selector_URing *selector, VALUE fiber, int descriptor, c
 	);
 }
 
-VALUE IO_Event_Selector_URing_io_read(VALUE self, VALUE fiber, VALUE io, VALUE buffer, VALUE _length, VALUE _offset) {
+#if RUBY_FIBER_SCHEDULER_VERSION >= 4
+struct io_read_locked_arguments {
+	struct IO_Event_Selector_URing *selector;
+	VALUE fiber;
+	VALUE io;
+	off_t from;
+	size_t offset;
+	size_t length;
+	bool positional;
+};
+
+static VALUE
+io_read_locked(void *base, size_t size, VALUE _arguments)
+{
+	struct io_read_locked_arguments *arguments = (struct io_read_locked_arguments *)_arguments;
+	
+	if (!IO_Event_Selector_valid_buffer_range(size, arguments->offset, arguments->length)) {
+		return rb_fiber_scheduler_io_result(-1, EINVAL);
+	} else if (arguments->length == 0) {
+		return rb_fiber_scheduler_io_result(0, 0);
+	}
+	
+	int descriptor = IO_Event_Selector_io_descriptor(arguments->io);
+	char *buffer = (char*)base + arguments->offset;
+	
+	// Avoid the submission and suspension overhead when the operation can
+	// complete immediately. The descriptor must be non-blocking for this
+	// optimistic syscall; if it would block, restore its original mode and
+	// submit the operation to io_uring instead.
+	int flags = IO_Event_Selector_nonblock_set(descriptor);
+	ssize_t result;
+	
+	if (arguments->positional) {
+		result = pread(descriptor, buffer, arguments->length, arguments->from);
+	} else {
+		result = read(descriptor, buffer, arguments->length);
+	}
+	
+	int error = errno;
+	IO_Event_Selector_nonblock_restore(descriptor, flags);
+	
+	if (result >= 0) {
+		return rb_fiber_scheduler_io_result(result, 0);
+	} else if (!IO_Event_try_again(error)) {
+		return rb_fiber_scheduler_io_result(-1, error);
+	}
+	
+	off_t from = arguments->positional ? arguments->from : io_seekable(descriptor);
+	
+	int completion = io_read(arguments->selector, arguments->fiber, descriptor, buffer, arguments->length, from);
+	if (completion < 0) {
+		return rb_fiber_scheduler_io_result(-1, -completion);
+	}
+	
+	return rb_fiber_scheduler_io_result(completion, 0);
+}
+#endif
+
+VALUE IO_Event_Selector_URing_io_read(VALUE self, VALUE fiber, VALUE io, VALUE buffer, VALUE _first, VALUE _second) {
 	struct IO_Event_Selector_URing *selector = NULL;
 	TypedData_Get_Struct(self, struct IO_Event_Selector_URing, &IO_Event_Selector_URing_Type, selector);
 	
+#if RUBY_FIBER_SCHEDULER_VERSION >= 4
+	struct io_read_locked_arguments arguments = {
+		.selector = selector,
+		.fiber = fiber,
+		.io = io,
+		.offset = NUM2SIZET(_first),
+		.length = NUM2SIZET(_second),
+		.positional = false,
+	};
+	
+	return rb_io_buffer_locked_for_writing(buffer, io_read_locked, (VALUE)&arguments);
+#else
 	void *base;
 	size_t size;
 	rb_io_buffer_get_bytes_for_writing(buffer, &base, &size);
 	
-	size_t length = NUM2SIZET(_length);
-	size_t offset = NUM2SIZET(_offset);
+	size_t length = NUM2SIZET(_first);
+	size_t offset = NUM2SIZET(_second);
 	size_t total = 0;
 	
 	// Ensure offset is within the bounds of the buffer to avoid size_t underflow and out-of-bounds pointer arithmetic on (char *)base + offset.
@@ -894,10 +1034,10 @@ VALUE IO_Event_Selector_URing_io_read(VALUE self, VALUE fiber, VALUE io, VALUE b
 	} else if (offset == size) {
 		return rb_fiber_scheduler_io_result(0, 0);
 	}
+#endif
 	
 	int descriptor = IO_Event_Selector_io_descriptor(io);
 	off_t from = io_seekable(descriptor);
-	
 	size_t maximum_size = size - offset;
 	
 	// Are we performing a non-blocking read?
@@ -933,8 +1073,10 @@ VALUE IO_Event_Selector_URing_io_read(VALUE self, VALUE fiber, VALUE io, VALUE b
 	}
 	
 	return rb_fiber_scheduler_io_result(total, 0);
+#endif
 }
 
+#if RUBY_FIBER_SCHEDULER_VERSION < 4
 static VALUE IO_Event_Selector_URing_io_read_compatible(int argc, VALUE *argv, VALUE self)
 {
 	rb_check_arity(argc, 4, 5);
@@ -947,19 +1089,32 @@ static VALUE IO_Event_Selector_URing_io_read_compatible(int argc, VALUE *argv, V
 	
 	return IO_Event_Selector_URing_io_read(self, argv[0], argv[1], argv[2], argv[3], _offset);
 }
+#endif
 
-VALUE IO_Event_Selector_URing_io_pread(VALUE self, VALUE fiber, VALUE io, VALUE buffer, VALUE _from, VALUE _length, VALUE _offset) {
+VALUE IO_Event_Selector_URing_io_pread(VALUE self, VALUE fiber, VALUE io, VALUE buffer, VALUE _from, VALUE _first, VALUE _second) {
 	struct IO_Event_Selector_URing *selector = NULL;
 	TypedData_Get_Struct(self, struct IO_Event_Selector_URing, &IO_Event_Selector_URing_Type, selector);
 	
+#if RUBY_FIBER_SCHEDULER_VERSION >= 4
+	struct io_read_locked_arguments arguments = {
+		.selector = selector,
+		.fiber = fiber,
+		.io = io,
+		.from = NUM2OFFT(_from),
+		.offset = NUM2SIZET(_first),
+		.length = NUM2SIZET(_second),
+		.positional = true,
+	};
+	
+	return rb_io_buffer_locked_for_writing(buffer, io_read_locked, (VALUE)&arguments);
+#else
 	void *base;
 	size_t size;
 	rb_io_buffer_get_bytes_for_writing(buffer, &base, &size);
 	
-	size_t length = NUM2SIZET(_length);
-	size_t offset = NUM2SIZET(_offset);
+	size_t length = NUM2SIZET(_first);
+	size_t offset = NUM2SIZET(_second);
 	size_t total = 0;
-	off_t from = NUM2OFFT(_from);
 	
 	// Ensure offset is within the bounds of the buffer to avoid size_t underflow and out-of-bounds pointer arithmetic on (char *)base + offset.
 	if (offset > size) {
@@ -967,9 +1122,9 @@ VALUE IO_Event_Selector_URing_io_pread(VALUE self, VALUE fiber, VALUE io, VALUE
 	} else if (offset == size) {
 		return rb_fiber_scheduler_io_result(0, 0);
 	}
-	
+#endif
+	off_t from = NUM2OFFT(_from);
 	int descriptor = IO_Event_Selector_io_descriptor(io);
-	
 	size_t maximum_size = size - offset;
 	while (maximum_size) {
 		int result = io_read(selector, fiber, descriptor, (char*)base+offset, maximum_size, from);
@@ -992,6 +1147,7 @@ VALUE IO_Event_Selector_URing_io_pread(VALUE self, VALUE fiber, VALUE io, VALUE
 	}
 	
 	return rb_fiber_scheduler_io_result(total, 0);
+#endif
 }
 
 #pragma mark - IO#write
@@ -1029,16 +1185,7 @@ io_write_ensure(VALUE _argument)
 	struct io_write_arguments *arguments = (struct io_write_arguments*)_argument;
 	struct IO_Event_Selector_URing *selector = arguments->selector;
 	
-	// If the operation is still in progress, cancel it:
-	if (arguments->waiting->completion) {
-		if (DEBUG) fprintf(stderr, "io_write_ensure:io_uring_prep_cancel(waiting=%p, completion=%p)\n", (void*)arguments->waiting, (void*)arguments->waiting->completion);
-		struct io_uring_sqe *sqe = io_get_sqe(selector);
-		io_uring_prep_cancel(sqe, (void*)arguments->waiting->completion, 0);
-		io_uring_sqe_set_data(sqe, NULL);
-		io_uring_submit_now(selector);
-	}
-	
-	IO_Event_Selector_URing_Waiting_cancel(arguments->waiting);
+	IO_Event_Selector_URing_Waiting_cancel_and_wait(selector, arguments->waiting);
 	
 	return Qnil;
 }
@@ -1068,16 +1215,86 @@ io_write(struct IO_Event_Selector_URing *selector, VALUE fiber, int descriptor,
 	);
 }
 
-VALUE IO_Event_Selector_URing_io_write(VALUE self, VALUE fiber, VALUE io, VALUE buffer, VALUE _length, VALUE _offset) {
+#if RUBY_FIBER_SCHEDULER_VERSION >= 4
+struct io_write_locked_arguments {
+	struct IO_Event_Selector_URing *selector;
+	VALUE fiber;
+	VALUE io;
+	off_t from;
+	size_t offset;
+	size_t length;
+	bool positional;
+};
+
+static VALUE
+io_write_locked(const void *base, size_t size, VALUE _arguments)
+{
+	struct io_write_locked_arguments *arguments = (struct io_write_locked_arguments *)_arguments;
+	
+	if (!IO_Event_Selector_valid_buffer_range(size, arguments->offset, arguments->length)) {
+		return rb_fiber_scheduler_io_result(-1, EINVAL);
+	} else if (arguments->length == 0) {
+		return rb_fiber_scheduler_io_result(0, 0);
+	}
+	
+	int descriptor = IO_Event_Selector_io_descriptor(arguments->io);
+	const char *buffer = (const char*)base + arguments->offset;
+	
+	// Avoid the submission and suspension overhead when the operation can
+	// complete immediately. The descriptor must be non-blocking for this
+	// optimistic syscall; if it would block, restore its original mode and
+	// submit the operation to io_uring instead.
+	int flags = IO_Event_Selector_nonblock_set(descriptor);
+	ssize_t result;
+	
+	if (arguments->positional) {
+		result = pwrite(descriptor, buffer, arguments->length, arguments->from);
+	} else {
+		result = write(descriptor, buffer, arguments->length);
+	}
+	
+	int error = errno;
+	IO_Event_Selector_nonblock_restore(descriptor, flags);
+	
+	if (result >= 0) {
+		return rb_fiber_scheduler_io_result(result, 0);
+	} else if (!IO_Event_try_again(error)) {
+		return rb_fiber_scheduler_io_result(-1, error);
+	}
+	
+	off_t from = arguments->positional ? arguments->from : io_seekable(descriptor);
+	
+	int completion = io_write(arguments->selector, arguments->fiber, descriptor, (char*)buffer, arguments->length, from);
+	if (completion < 0) {
+		return rb_fiber_scheduler_io_result(-1, -completion);
+	}
+	
+	return rb_fiber_scheduler_io_result(completion, 0);
+}
+#endif
+
+VALUE IO_Event_Selector_URing_io_write(VALUE self, VALUE fiber, VALUE io, VALUE buffer, VALUE _first, VALUE _second) {
 	struct IO_Event_Selector_URing *selector = NULL;
 	TypedData_Get_Struct(self, struct IO_Event_Selector_URing, &IO_Event_Selector_URing_Type, selector);
 	
+#if RUBY_FIBER_SCHEDULER_VERSION >= 4
+	struct io_write_locked_arguments arguments = {
+		.selector = selector,
+		.fiber = fiber,
+		.io = io,
+		.offset = NUM2SIZET(_first),
+		.length = NUM2SIZET(_second),
+		.positional = false,
+	};
+	
+	return rb_io_buffer_locked_for_reading(buffer, io_write_locked, (VALUE)&arguments);
+#else
 	const void *base;
 	size_t size;
 	rb_io_buffer_get_bytes_for_reading(buffer, &base, &size);
 	
-	size_t length = NUM2SIZET(_length);
-	size_t offset = NUM2SIZET(_offset);
+	size_t length = NUM2SIZET(_first);
+	size_t offset = NUM2SIZET(_second);
 	size_t total = 0;
 	
 	if (length > size) {
@@ -1090,10 +1307,10 @@ VALUE IO_Event_Selector_URing_io_write(VALUE self, VALUE fiber, VALUE io, VALUE
 	} else if (offset == size) {
 		return rb_fiber_scheduler_io_result(0, 0);
 	}
+#endif
 	
 	int descriptor = IO_Event_Selector_io_descriptor(io);
 	off_t from = io_seekable(descriptor);
-	
 	size_t maximum_size = size - offset;
 	while (maximum_size) {
 		int result = io_write(selector, fiber, descriptor, (char*)base+offset, maximum_size, from);
@@ -1115,8 +1332,10 @@ VALUE IO_Event_Selector_URing_io_write(VALUE self, VALUE fiber, VALUE io, VALUE
 	}
 	
 	return rb_fiber_scheduler_io_result(total, 0);
+#endif
 }
 
+#if RUBY_FIBER_SCHEDULER_VERSION < 4
 static VALUE IO_Event_Selector_URing_io_write_compatible(int argc, VALUE *argv, VALUE self)
 {
 	rb_check_arity(argc, 4, 5);
@@ -1129,19 +1348,32 @@ static VALUE IO_Event_Selector_URing_io_write_compatible(int argc, VALUE *argv,
 	
 	return IO_Event_Selector_URing_io_write(self, argv[0], argv[1], argv[2], argv[3], _offset);
 }
+#endif
 
-VALUE IO_Event_Selector_URing_io_pwrite(VALUE self, VALUE fiber, VALUE io, VALUE buffer, VALUE _from, VALUE _length, VALUE _offset) {
+VALUE IO_Event_Selector_URing_io_pwrite(VALUE self, VALUE fiber, VALUE io, VALUE buffer, VALUE _from, VALUE _first, VALUE _second) {
 	struct IO_Event_Selector_URing *selector = NULL;
 	TypedData_Get_Struct(self, struct IO_Event_Selector_URing, &IO_Event_Selector_URing_Type, selector);
 	
+#if RUBY_FIBER_SCHEDULER_VERSION >= 4
+	struct io_write_locked_arguments arguments = {
+		.selector = selector,
+		.fiber = fiber,
+		.io = io,
+		.from = NUM2OFFT(_from),
+		.offset = NUM2SIZET(_first),
+		.length = NUM2SIZET(_second),
+		.positional = true,
+	};
+	
+	return rb_io_buffer_locked_for_reading(buffer, io_write_locked, (VALUE)&arguments);
+#else
 	const void *base;
 	size_t size;
 	rb_io_buffer_get_bytes_for_reading(buffer, &base, &size);
 	
-	size_t length = NUM2SIZET(_length);
-	size_t offset = NUM2SIZET(_offset);
+	size_t length = NUM2SIZET(_first);
+	size_t offset = NUM2SIZET(_second);
 	size_t total = 0;
-	off_t from = NUM2OFFT(_from);
 	
 	if (length > size) {
 		rb_raise(rb_eRuntimeError, "Length exceeds size of buffer!");
@@ -1153,9 +1385,9 @@ VALUE IO_Event_Selector_URing_io_pwrite(VALUE self, VALUE fiber, VALUE io, VALUE
 	} else if (offset == size) {
 		return rb_fiber_scheduler_io_result(0, 0);
 	}
-	
+#endif
+	off_t from = NUM2OFFT(_from);
 	int descriptor = IO_Event_Selector_io_descriptor(io);
-	
 	size_t maximum_size = size - offset;
 	while (maximum_size) {
 		int result = io_write(selector, fiber, descriptor, (char*)base+offset, maximum_size, from);
@@ -1178,6 +1410,7 @@ VALUE IO_Event_Selector_URing_io_pwrite(VALUE self, VALUE fiber, VALUE io, VALUE
 	}
 	
 	return rb_fiber_scheduler_io_result(total, 0);
+#endif
 }
 
 #endif
@@ -1329,6 +1562,13 @@ unsigned select_process_completions(struct IO_Event_Selector_URing *selector) {
 			continue;
 		}
 		
+		if (cqe->user_data & IO_EVENT_SELECTOR_URING_CANCELLATION_TAG) {
+			struct IO_Event_Selector_URing_Completion *completion = IO_Event_Selector_URing_Completion_from_cancellation_data(cqe->user_data);
+			IO_Event_Selector_URing_Completion_cancellation_complete(selector, completion);
+			io_uring_cq_advance(ring, 1);
+			continue;
+		}
+		
 		struct IO_Event_Selector_URing_Completion *completion = (void*)cqe->user_data;
 		struct IO_Event_Selector_URing_Waiting *waiting = completion->waiting;
 		
@@ -1343,13 +1583,11 @@ unsigned select_process_completions(struct IO_Event_Selector_URing *selector) {
 		
 		VALUE fiber = 0;
 		if (waiting && waiting->fiber) {
-			assert(waiting->result != -ECANCELED);
-			
 			fiber = waiting->fiber;
 		}
 
 		// This marks the waiting operation as "complete":
-		IO_Event_Selector_URing_Completion_release(selector, completion);
+		IO_Event_Selector_URing_Completion_complete(selector, completion);
 		
 		if (fiber) {
 			IO_Event_Selector_loop_resume(&selector->backend, fiber, 0, NULL);
@@ -1511,8 +1749,13 @@ void Init_IO_Event_Selector_URing(VALUE IO_Event_Selector) {
 	rb_define_method(IO_Event_Selector_URing, "io_wait", IO_Event_Selector_URing_io_wait, 3);
 	
 #ifdef HAVE_RUBY_IO_BUFFER_H
+#if RUBY_FIBER_SCHEDULER_VERSION >= 4
+	rb_define_method(IO_Event_Selector_URing, "io_read", IO_Event_Selector_URing_io_read, 5);
+	rb_define_method(IO_Event_Selector_URing, "io_write", IO_Event_Selector_URing_io_write, 5);
+#else
 	rb_define_method(IO_Event_Selector_URing, "io_read", IO_Event_Selector_URing_io_read_compatible, -1);
 	rb_define_method(IO_Event_Selector_URing, "io_write", IO_Event_Selector_URing_io_write_compatible, -1);
+#endif
 	rb_define_method(IO_Event_Selector_URing, "io_pread", IO_Event_Selector_URing_io_pread, 6);
 	rb_define_method(IO_Event_Selector_URing, "io_pwrite", IO_Event_Selector_URing_io_pwrite, 6);
 #endif

data/ext/io/event/worker_pool.c

@@ -306,15 +306,54 @@ static VALUE worker_pool_initialize(int argc, VALUE *argv, VALUE self) {
 	return self;
 }
 
-static VALUE worker_pool_work_begin(VALUE _work) {
+static VALUE worker_pool_work_block(VALUE _work) {
 	struct IO_Event_WorkerPool_Work *work = (void*)_work;
 
-	if (DEBUG) fprintf(stderr, "worker_pool_work_begin:rb_fiber_scheduler_block work=%p\n", work);
+	if (DEBUG) fprintf(stderr, "worker_pool_work_block:rb_fiber_scheduler_block work=%p\n", work);
 	rb_fiber_scheduler_block(work->scheduler, work->blocker, Qnil);
 
 	return Qnil;
 }
 
+static VALUE worker_pool_work_wait(VALUE _work) {
+	struct IO_Event_WorkerPool_Work *work = (void*)_work;
+
+	while (true) {
+		worker_pool_work_block(_work);
+		if (DEBUG) fprintf(stderr, "-- worker_pool_work_wait:work completed=%d\n", work->completed);
+		
+		if (work->completed) {
+			break;
+		}
+		
+		if (DEBUG) fprintf(stderr, "worker_pool_work_wait:rb_fiber_scheduler_blocking_operation_cancel\n");
+		rb_fiber_scheduler_blocking_operation_cancel(work->blocking_operation);
+	}
+
+	return Qtrue;
+}
+
+static VALUE worker_pool_work_ensure(VALUE _work) {
+	struct IO_Event_WorkerPool_Work *work = (void*)_work;
+
+	while (!work->completed) {
+		if (DEBUG) fprintf(stderr, "worker_pool_work_ensure:rb_fiber_scheduler_blocking_operation_cancel\n");
+		rb_fiber_scheduler_blocking_operation_cancel(work->blocking_operation);
+		
+		int state = 0;
+		rb_protect(worker_pool_work_block, _work, &state);
+		if (DEBUG) fprintf(stderr, "-- worker_pool_work_ensure:work completed=%d, state=%d\n", work->completed, state);
+		
+		if (state) {
+			// Ignore errors raised while waiting for cancellation to complete. rb_ensure
+			// will restore and rethrow the original control-flow state after this returns.
+			rb_set_errinfo(Qnil);
+		}
+	}
+
+	return Qnil;
+}
+
 // Ruby method to submit work and wait for completion
 static VALUE worker_pool_call(VALUE self, VALUE _blocking_operation) {
 	struct IO_Event_WorkerPool *pool;
@@ -357,38 +396,10 @@ static VALUE worker_pool_call(VALUE self, VALUE _blocking_operation) {
 	pthread_cond_signal(&pool->work_available);
 	pthread_mutex_unlock(&pool->mutex);
 	
-	// Block the current fiber until work is completed:
-	int state = 0;
-	while (true) {
-		int current_state = 0;
-		rb_protect(worker_pool_work_begin, (VALUE)&work, &current_state);
-		if (DEBUG) fprintf(stderr, "-- worker_pool_call:work completed=%d, current_state=%d, state=%d\n", work.completed, current_state, state);
-		
-		// Store the first exception state:
-		if (!state) {
-			state = current_state;
-		}
-		
-		// If the work is still in the queue, we must wait for a worker to complete it (even if cancelled):
-		if (work.completed) {
-			// The work was completed, we can exit the loop:
-			break;
-		} else {
-			if (DEBUG) fprintf(stderr, "worker_pool_call:rb_fiber_scheduler_blocking_operation_cancel\n");
-			// Ensure the blocking operation is cancelled:
-			rb_fiber_scheduler_blocking_operation_cancel(blocking_operation);
-			
-			// The work was not completed, we need to wait for it to be completed, so we go around the loop again.
-		}
-	}
-	
-	if (DEBUG) fprintf(stderr, "<- worker_pool_call:work completed=%d, state=%d\n", work.completed, state);
-	
-	if (state) {
-		rb_jump_tag(state);
-	} else {
-		return Qtrue;
-	}
+	// Block the current fiber until work is completed. If blocking exits via an
+	// exception or another non-local jump, ensure the work is cancelled and fully
+	// drained before Ruby restores and rethrows the original control-flow state.
+	return rb_ensure(worker_pool_work_wait, (VALUE)&work, worker_pool_work_ensure, (VALUE)&work);
 }
 
 static VALUE worker_pool_allocate(VALUE klass) {

data/lib/io/event/debug/selector.rb

@@ -177,15 +177,15 @@ module IO::Event
 			end
 			
 			# Read from the given IO, forwarded to the underlying selector.
-			def io_read(fiber, io, buffer, length, offset = 0)
-				log("Reading from IO #{io.inspect} with buffer #{buffer}; length #{length} offset #{offset}")
-				@selector.io_read(fiber, io, buffer, length, offset)
+			def io_read(fiber, io, buffer, *arguments)
+				log("Reading from IO #{io.inspect} with buffer #{buffer}; arguments #{arguments.inspect}")
+				@selector.io_read(fiber, io, buffer, *arguments)
 			end
 			
 			# Write to the given IO, forwarded to the underlying selector.
-			def io_write(fiber, io, buffer, length, offset = 0)
-				log("Writing to IO #{io.inspect} with buffer #{buffer}; length #{length} offset #{offset}")
-				@selector.io_write(fiber, io, buffer, length, offset)
+			def io_write(fiber, io, buffer, *arguments)
+				log("Writing to IO #{io.inspect} with buffer #{buffer}; arguments #{arguments.inspect}")
+				@selector.io_write(fiber, io, buffer, *arguments)
 			end
 			
 			# Forward the given method to the underlying selector.