io-event 1.2.3 → 1.3.0

data/ext/io/event/selector/uring.c

@@ -20,6 +20,8 @@
 
 #include "uring.h"
 #include "selector.h"
+#include "list.h"
+#include "array.h"
 
 #include <liburing.h>
 #include <poll.h>
@@ -32,7 +34,6 @@
 
 enum {
 	DEBUG = 0,
-	DEBUG_IO_READ = 0,
 };
 
 static VALUE IO_Event_Selector_URing = Qnil;
@@ -41,45 +42,109 @@ enum {URING_ENTRIES = 64};
 
 #pragma mark - Data Type
 
-struct IO_Event_Selector_URing {
+struct IO_Event_Selector_URing
+{
 	struct IO_Event_Selector backend;
 	struct io_uring ring;
 	size_t pending;
 	int blocked;
+	
+	struct IO_Event_Array completions;
+	struct IO_Event_List free_list;
 };
 
-void IO_Event_Selector_URing_Type_mark(void *_data)
+struct IO_Event_Selector_URing_Completion;
+
+struct IO_Event_Selector_URing_Waiting
+{
+	struct IO_Event_Selector_URing_Completion *completion;
+	
+	VALUE fiber;
+	
+	// The result of the operation.
+	int32_t	result;
+	
+	// Any associated flags.
+	uint32_t flags;
+};
+
+struct IO_Event_Selector_URing_Completion
+{
+	struct IO_Event_List list;
+	
+	struct IO_Event_Selector_URing_Waiting *waiting;
+};
+
+static
+void IO_Event_Selector_URing_Completion_mark(void *_completion)
+{
+	struct IO_Event_Selector_URing_Completion *completion = _completion;
+	
+	if (completion->waiting) {
+		rb_gc_mark_movable(completion->waiting->fiber);
+	}
+}
+
+void IO_Event_Selector_URing_Type_mark(void *_selector)
 {
-	struct IO_Event_Selector_URing *data = _data;
-	IO_Event_Selector_mark(&data->backend);
+	struct IO_Event_Selector_URing *selector = _selector;
+	IO_Event_Selector_mark(&selector->backend);
+	IO_Event_Array_each(&selector->completions, IO_Event_Selector_URing_Completion_mark);
 }
 
 static
-void close_internal(struct IO_Event_Selector_URing *data) {
-	if (data->ring.ring_fd >= 0) {
-		io_uring_queue_exit(&data->ring);
-		data->ring.ring_fd = -1;
+void IO_Event_Selector_URing_Completion_compact(void *_completion)
+{
+	struct IO_Event_Selector_URing_Completion *completion = _completion;
+	
+	if (completion->waiting) {
+		completion->waiting->fiber = rb_gc_location(completion->waiting->fiber);
 	}
 }
 
-void IO_Event_Selector_URing_Type_free(void *_data)
+void IO_Event_Selector_URing_Type_compact(void *_selector)
+{
+	struct IO_Event_Selector_URing *selector = _selector;
+	IO_Event_Selector_compact(&selector->backend);
+	IO_Event_Array_each(&selector->completions, IO_Event_Selector_URing_Completion_compact);
+}
+
+static
+void close_internal(struct IO_Event_Selector_URing *selector)
 {
-	struct IO_Event_Selector_URing *data = _data;
+	if (selector->ring.ring_fd >= 0) {
+		io_uring_queue_exit(&selector->ring);
+		selector->ring.ring_fd = -1;
+	}
+}
+
+static
+void IO_Event_Selector_URing_Type_free(void *_selector)
+{
+	struct IO_Event_Selector_URing *selector = _selector;
+	
+	close_internal(selector);
 	
-	close_internal(data);
+	IO_Event_Array_free(&selector->completions);
 	
-	free(data);
+	free(selector);
 }
 
-size_t IO_Event_Selector_URing_Type_size(const void *data)
+static
+size_t IO_Event_Selector_URing_Type_size(const void *_selector)
 {
-	return sizeof(struct IO_Event_Selector_URing);
+	const struct IO_Event_Selector_URing *selector = _selector;
+	
+	return sizeof(struct IO_Event_Selector_URing)
+		+ IO_Event_Array_memory_size(&selector->completions)
+	;
 }
 
 static const rb_data_type_t IO_Event_Selector_URing_Type = {
 	.wrap_struct_name = "IO_Event::Backend::URing",
 	.function = {
 		.dmark = IO_Event_Selector_URing_Type_mark,
+		.dcompact = IO_Event_Selector_URing_Type_compact,
 		.dfree = IO_Event_Selector_URing_Type_free,
 		.dsize = IO_Event_Selector_URing_Type_size,
 	},
@@ -87,15 +152,79 @@ static const rb_data_type_t IO_Event_Selector_URing_Type = {
 	.flags = RUBY_TYPED_FREE_IMMEDIATELY,
 };
 
+inline static
+struct IO_Event_Selector_URing_Completion * IO_Event_Selector_URing_Completion_acquire(struct IO_Event_Selector_URing *selector, struct IO_Event_Selector_URing_Waiting *waiting)
+{
+	struct IO_Event_Selector_URing_Completion *completion = NULL;
+	
+	if (!IO_Event_List_empty(&selector->free_list)) {
+		completion = (struct IO_Event_Selector_URing_Completion*)selector->free_list.tail;
+		IO_Event_List_pop(&completion->list);
+	} else {
+		completion = IO_Event_Array_push(&selector->completions);
+		IO_Event_List_clear(&completion->list);
+	}
+	
+	waiting->completion = completion;
+	completion->waiting = waiting;
+	
+	return completion;
+}
+
+inline static
+void IO_Event_Selector_URing_Completion_cancel(struct IO_Event_Selector_URing_Completion *completion)
+{
+	if (completion->waiting) {
+		completion->waiting->completion = NULL;
+		completion->waiting = NULL;
+	}
+}
+
+inline static
+void IO_Event_Selector_URing_Completion_release(struct IO_Event_Selector_URing *selector, struct IO_Event_Selector_URing_Completion *completion)
+{
+	IO_Event_Selector_URing_Completion_cancel(completion);
+	IO_Event_List_prepend(&selector->free_list, &completion->list);
+}
+
+inline static
+void IO_Event_Selector_URing_Waiting_cancel(struct IO_Event_Selector_URing *selector, struct IO_Event_Selector_URing_Waiting *waiting)
+{
+	if (waiting->completion) {
+		waiting->completion->waiting = NULL;
+		waiting->completion = NULL;
+	}
+	
+	waiting->fiber = 0;
+}
+
+void IO_Event_Selector_URing_Completion_initialize(void *element)
+{
+	struct IO_Event_Selector_URing_Completion *completion = element;
+	IO_Event_List_initialize(&completion->list);
+}
+
+void IO_Event_Selector_URing_Completion_free(void *element)
+{
+	struct IO_Event_Selector_URing_Completion *completion = element;
+	IO_Event_Selector_URing_Completion_cancel(completion);
+}
+
 VALUE IO_Event_Selector_URing_allocate(VALUE self) {
-	struct IO_Event_Selector_URing *data = NULL;
-	VALUE instance = TypedData_Make_Struct(self, struct IO_Event_Selector_URing, &IO_Event_Selector_URing_Type, data);
+	struct IO_Event_Selector_URing *selector = NULL;
+	VALUE instance = TypedData_Make_Struct(self, struct IO_Event_Selector_URing, &IO_Event_Selector_URing_Type, selector);
 	
-	IO_Event_Selector_initialize(&data->backend, Qnil);
-	data->ring.ring_fd = -1;
+	IO_Event_Selector_initialize(&selector->backend, Qnil);
+	selector->ring.ring_fd = -1;
 	
-	data->pending = 0;
-	data->blocked = 0;
+	selector->pending = 0;
+	selector->blocked = 0;
+	
+	IO_Event_List_initialize(&selector->free_list);
+	
+	selector->completions.element_initialize = IO_Event_Selector_URing_Completion_initialize;
+	selector->completions.element_free = IO_Event_Selector_URing_Completion_free;
+	IO_Event_Array_allocate(&selector->completions, 1024, sizeof(struct IO_Event_Selector_URing_Completion));
 	
 	return instance;
 }
@@ -103,100 +232,100 @@ VALUE IO_Event_Selector_URing_allocate(VALUE self) {
 #pragma mark - Methods
 
 VALUE IO_Event_Selector_URing_initialize(VALUE self, VALUE loop) {
-	struct IO_Event_Selector_URing *data = NULL;
-	TypedData_Get_Struct(self, struct IO_Event_Selector_URing, &IO_Event_Selector_URing_Type, data);
+	struct IO_Event_Selector_URing *selector = NULL;
+	TypedData_Get_Struct(self, struct IO_Event_Selector_URing, &IO_Event_Selector_URing_Type, selector);
 	
-	IO_Event_Selector_initialize(&data->backend, loop);
-	int result = io_uring_queue_init(URING_ENTRIES, &data->ring, 0);
+	IO_Event_Selector_initialize(&selector->backend, loop);
+	int result = io_uring_queue_init(URING_ENTRIES, &selector->ring, 0);
 	
 	if (result < 0) {
 		rb_syserr_fail(-result, "IO_Event_Selector_URing_initialize:io_uring_queue_init");
 	}
 	
-	rb_update_max_fd(data->ring.ring_fd);
+	rb_update_max_fd(selector->ring.ring_fd);
 	
 	return self;
 }
 
 VALUE IO_Event_Selector_URing_loop(VALUE self) {
-	struct IO_Event_Selector_URing *data = NULL;
-	TypedData_Get_Struct(self, struct IO_Event_Selector_URing, &IO_Event_Selector_URing_Type, data);
+	struct IO_Event_Selector_URing *selector = NULL;
+	TypedData_Get_Struct(self, struct IO_Event_Selector_URing, &IO_Event_Selector_URing_Type, selector);
 	
-	return data->backend.loop;
+	return selector->backend.loop;
 }
 
 VALUE IO_Event_Selector_URing_close(VALUE self) {
-	struct IO_Event_Selector_URing *data = NULL;
-	TypedData_Get_Struct(self, struct IO_Event_Selector_URing, &IO_Event_Selector_URing_Type, data);
+	struct IO_Event_Selector_URing *selector = NULL;
+	TypedData_Get_Struct(self, struct IO_Event_Selector_URing, &IO_Event_Selector_URing_Type, selector);
 	
-	close_internal(data);
+	close_internal(selector);
 	
 	return Qnil;
 }
 
 VALUE IO_Event_Selector_URing_transfer(VALUE self)
 {
-	struct IO_Event_Selector_URing *data = NULL;
-	TypedData_Get_Struct(self, struct IO_Event_Selector_URing, &IO_Event_Selector_URing_Type, data);
+	struct IO_Event_Selector_URing *selector = NULL;
+	TypedData_Get_Struct(self, struct IO_Event_Selector_URing, &IO_Event_Selector_URing_Type, selector);
 	
-	return IO_Event_Selector_fiber_transfer(data->backend.loop, 0, NULL);
+	return IO_Event_Selector_fiber_transfer(selector->backend.loop, 0, NULL);
 }
 
 VALUE IO_Event_Selector_URing_resume(int argc, VALUE *argv, VALUE self)
 {
-	struct IO_Event_Selector_URing *data = NULL;
-	TypedData_Get_Struct(self, struct IO_Event_Selector_URing, &IO_Event_Selector_URing_Type, data);
+	struct IO_Event_Selector_URing *selector = NULL;
+	TypedData_Get_Struct(self, struct IO_Event_Selector_URing, &IO_Event_Selector_URing_Type, selector);
 	
-	return IO_Event_Selector_resume(&data->backend, argc, argv);
+	return IO_Event_Selector_resume(&selector->backend, argc, argv);
 }
 
 VALUE IO_Event_Selector_URing_yield(VALUE self)
 {
-	struct IO_Event_Selector_URing *data = NULL;
-	TypedData_Get_Struct(self, struct IO_Event_Selector_URing, &IO_Event_Selector_URing_Type, data);
+	struct IO_Event_Selector_URing *selector = NULL;
+	TypedData_Get_Struct(self, struct IO_Event_Selector_URing, &IO_Event_Selector_URing_Type, selector);
 	
-	return IO_Event_Selector_yield(&data->backend);
+	return IO_Event_Selector_yield(&selector->backend);
 }
 
 VALUE IO_Event_Selector_URing_push(VALUE self, VALUE fiber)
 {
-	struct IO_Event_Selector_URing *data = NULL;
-	TypedData_Get_Struct(self, struct IO_Event_Selector_URing, &IO_Event_Selector_URing_Type, data);
+	struct IO_Event_Selector_URing *selector = NULL;
+	TypedData_Get_Struct(self, struct IO_Event_Selector_URing, &IO_Event_Selector_URing_Type, selector);
 	
-	IO_Event_Selector_queue_push(&data->backend, fiber);
+	IO_Event_Selector_queue_push(&selector->backend, fiber);
 	
 	return Qnil;
 }
 
 VALUE IO_Event_Selector_URing_raise(int argc, VALUE *argv, VALUE self)
 {
-	struct IO_Event_Selector_URing *data = NULL;
-	TypedData_Get_Struct(self, struct IO_Event_Selector_URing, &IO_Event_Selector_URing_Type, data);
+	struct IO_Event_Selector_URing *selector = NULL;
+	TypedData_Get_Struct(self, struct IO_Event_Selector_URing, &IO_Event_Selector_URing_Type, selector);
 	
-	return IO_Event_Selector_raise(&data->backend, argc, argv);
+	return IO_Event_Selector_raise(&selector->backend, argc, argv);
 }
 	
 VALUE IO_Event_Selector_URing_ready_p(VALUE self) {
-	struct IO_Event_Selector_URing *data = NULL;
-	TypedData_Get_Struct(self, struct IO_Event_Selector_URing, &IO_Event_Selector_URing_Type, data);
+	struct IO_Event_Selector_URing *selector = NULL;
+	TypedData_Get_Struct(self, struct IO_Event_Selector_URing, &IO_Event_Selector_URing_Type, selector);
 	
-	return data->backend.ready ? Qtrue : Qfalse;
+	return selector->backend.ready ? Qtrue : Qfalse;
 }
 
 #pragma mark - Submission Queue
 
 // Flush the submission queue if pending operations are present.
 static
-int io_uring_submit_flush(struct IO_Event_Selector_URing *data) {
-	if (data->pending) {
-		if (DEBUG) fprintf(stderr, "io_uring_submit_flush(pending=%ld)\n", data->pending);
+int io_uring_submit_flush(struct IO_Event_Selector_URing *selector) {
+	if (selector->pending) {
+		if (DEBUG) fprintf(stderr, "io_uring_submit_flush(pending=%ld)\n", selector->pending);
 		
 		// Try to submit:
-		int result = io_uring_submit(&data->ring);
+		int result = io_uring_submit(&selector->ring);
 		
 		if (result >= 0) {
 			// If it was submitted, reset pending count:
-			data->pending = 0;
+			selector->pending = 0;
 		} else if (result != -EBUSY && result != -EAGAIN) {
 			rb_syserr_fail(-result, "io_uring_submit_flush:io_uring_submit");
 		}
@@ -209,19 +338,19 @@ int io_uring_submit_flush(struct IO_Event_Selector_URing *data) {
 
 // Immediately flush the submission queue, yielding to the event loop if it was not successful.
 static
-int io_uring_submit_now(struct IO_Event_Selector_URing *data) {
-	if (DEBUG && data->pending) fprintf(stderr, "io_uring_submit_now(pending=%ld)\n", data->pending);
+int io_uring_submit_now(struct IO_Event_Selector_URing *selector) {
+	if (DEBUG && selector->pending) fprintf(stderr, "io_uring_submit_now(pending=%ld)\n", selector->pending);
 
 	while (true) {
-		int result = io_uring_submit(&data->ring);
+		int result = io_uring_submit(&selector->ring);
 		
 		if (result >= 0) {
-			data->pending = 0;
+			selector->pending = 0;
 			return result;
 		}
 		
 		if (result == -EBUSY || result == -EAGAIN) {
-			IO_Event_Selector_yield(&data->backend);
+			IO_Event_Selector_yield(&selector->backend);
 		} else {
 			rb_syserr_fail(-result, "io_uring_submit_now:io_uring_submit");
 		}
@@ -230,20 +359,20 @@ int io_uring_submit_now(struct IO_Event_Selector_URing *data) {
 
 // Submit a pending operation. This does not submit the operation immediately, but instead defers it to the next call to `io_uring_submit_flush` or `io_uring_submit_now`. This is useful for operations that are not urgent, but should be used with care as it can lead to a deadlock if the submission queue is not flushed.
 static
-void io_uring_submit_pending(struct IO_Event_Selector_URing *data) {
-	data->pending += 1;
+void io_uring_submit_pending(struct IO_Event_Selector_URing *selector) {
+	selector->pending += 1;
 	
-	if (DEBUG) fprintf(stderr, "io_uring_submit_pending(ring=%p, pending=%ld)\n", &data->ring, data->pending);
+	if (DEBUG) fprintf(stderr, "io_uring_submit_pending(ring=%p, pending=%ld)\n", &selector->ring, selector->pending);
 }
 
-struct io_uring_sqe * io_get_sqe(struct IO_Event_Selector_URing *data) {
-	struct io_uring_sqe *sqe = io_uring_get_sqe(&data->ring);
+struct io_uring_sqe * io_get_sqe(struct IO_Event_Selector_URing *selector) {
+	struct io_uring_sqe *sqe = io_uring_get_sqe(&selector->ring);
 	
 	while (sqe == NULL) {
 		// The submit queue is full, we need to drain it:	
-		io_uring_submit_now(data);
+		io_uring_submit_now(selector);
 		
-		sqe = io_uring_get_sqe(&data->ring);
+		sqe = io_uring_get_sqe(&selector->ring);
 	}
 	
 	return sqe;
@@ -252,9 +381,10 @@ struct io_uring_sqe * io_get_sqe(struct IO_Event_Selector_URing *data) {
 #pragma mark - Process.wait
 
 struct process_wait_arguments {
-	struct IO_Event_Selector_URing *data;
+	struct IO_Event_Selector_URing *selector;
+	struct IO_Event_Selector_URing_Waiting *waiting;
+	
 	pid_t pid;
-	int flags;
 	int descriptor;
 };
 
@@ -262,9 +392,13 @@ static
 VALUE process_wait_transfer(VALUE _arguments) {
 	struct process_wait_arguments *arguments = (struct process_wait_arguments *)_arguments;
 	
-	IO_Event_Selector_fiber_transfer(arguments->data->backend.loop, 0, NULL);
+	IO_Event_Selector_fiber_transfer(arguments->selector->backend.loop, 0, NULL);
 	
-	return IO_Event_Selector_process_status_wait(arguments->pid);
+	if (arguments->waiting->result) {
+		return IO_Event_Selector_process_status_wait(arguments->pid);
+	} else {
+		return Qfalse;
+	}
 }
 
 static
@@ -273,29 +407,43 @@ VALUE process_wait_ensure(VALUE _arguments) {
 	
 	close(arguments->descriptor);
 	
+	IO_Event_Selector_URing_Waiting_cancel(arguments->selector, arguments->waiting);
+	
 	return Qnil;
 }
 
-VALUE IO_Event_Selector_URing_process_wait(VALUE self, VALUE fiber, VALUE pid, VALUE flags) {
-	struct IO_Event_Selector_URing *data = NULL;
-	TypedData_Get_Struct(self, struct IO_Event_Selector_URing, &IO_Event_Selector_URing_Type, data);
+VALUE IO_Event_Selector_URing_process_wait(VALUE self, VALUE fiber, VALUE _pid, VALUE _flags) {
+	struct IO_Event_Selector_URing *selector = NULL;
+	TypedData_Get_Struct(self, struct IO_Event_Selector_URing, &IO_Event_Selector_URing_Type, selector);
+	
+	pid_t pid = NUM2PIDT(_pid);
+	
+	int descriptor = pidfd_open(pid, 0);
+	if (descriptor < 0) {
+		rb_syserr_fail(errno, "IO_Event_Selector_URing_process_wait:pidfd_open");
+	}
+	rb_update_max_fd(descriptor);
+	
+	struct IO_Event_Selector_URing_Waiting waiting = {
+		.fiber = fiber,
+	};
+	
+	struct IO_Event_Selector_URing_Completion *completion = IO_Event_Selector_URing_Completion_acquire(selector, &waiting);
 	
 	struct process_wait_arguments process_wait_arguments = {
-		.data = data,
-		.pid = NUM2PIDT(pid),
-		.flags = NUM2INT(flags),
+		.selector = selector,
+		.waiting = &waiting,
+		.pid = pid,
+		.descriptor = descriptor,
 	};
 	
-	process_wait_arguments.descriptor = pidfd_open(process_wait_arguments.pid, 0);
-	rb_update_max_fd(process_wait_arguments.descriptor);
+	struct io_uring_sqe *sqe = io_get_sqe(selector);
 	
-	struct io_uring_sqe *sqe = io_get_sqe(data);
-
 	if (DEBUG) fprintf(stderr, "IO_Event_Selector_URing_process_wait:io_uring_prep_poll_add(%p)\n", (void*)fiber);
-	io_uring_prep_poll_add(sqe, process_wait_arguments.descriptor, POLLIN|POLLHUP|POLLERR);
-	io_uring_sqe_set_data(sqe, (void*)fiber);
-	io_uring_submit_pending(data);
-
+	io_uring_prep_poll_add(sqe, descriptor, POLLIN|POLLHUP|POLLERR);
+	io_uring_sqe_set_data(sqe, completion);
+	io_uring_submit_pending(selector);
+	
 	return rb_ensure(process_wait_transfer, (VALUE)&process_wait_arguments, process_wait_ensure, (VALUE)&process_wait_arguments);
 }
 
@@ -328,70 +476,72 @@ int events_from_poll_flags(short flags) {
 }
 
 struct io_wait_arguments {
-	struct IO_Event_Selector_URing *data;
-	VALUE fiber;
+	struct IO_Event_Selector_URing *selector;
+	struct IO_Event_Selector_URing_Waiting *waiting;
 	short flags;
 };
 
 static
-VALUE io_wait_rescue(VALUE _arguments, VALUE exception) {
+VALUE io_wait_ensure(VALUE _arguments) {
 	struct io_wait_arguments *arguments = (struct io_wait_arguments *)_arguments;
-	struct IO_Event_Selector_URing *data = arguments->data;
 	
-	struct io_uring_sqe *sqe = io_get_sqe(data);
+	// We may want to consider cancellation. Be aware that the order of operations is important here:
+	// io_uring_prep_cancel(sqe, (void*)arguments->waiting, 0);
+	// io_uring_sqe_set_data(sqe, NULL);
+	// io_uring_submit_now(selector);
 	
-	if (DEBUG) fprintf(stderr, "io_wait_rescue:io_uring_prep_poll_remove(%p)\n", (void*)arguments->fiber);
+	IO_Event_Selector_URing_Waiting_cancel(arguments->selector, arguments->waiting);
 	
-	io_uring_prep_poll_remove(sqe, (uintptr_t)arguments->fiber);
-	io_uring_sqe_set_data(sqe, NULL);
-	io_uring_submit_now(data);
-
-	rb_exc_raise(exception);
+	return Qnil;
 };
 
 static
 VALUE io_wait_transfer(VALUE _arguments) {
 	struct io_wait_arguments *arguments = (struct io_wait_arguments *)_arguments;
-	struct IO_Event_Selector_URing *data = arguments->data;
-
-	VALUE result = IO_Event_Selector_fiber_transfer(data->backend.loop, 0, NULL);
-	if (DEBUG) fprintf(stderr, "io_wait:IO_Event_Selector_fiber_transfer -> %d\n", RB_NUM2INT(result));
-
-	if (!RTEST(result)) {
+	struct IO_Event_Selector_URing *selector = arguments->selector;
+	
+	IO_Event_Selector_fiber_transfer(selector->backend.loop, 0, NULL);
+	
+	if (arguments->waiting->result) {
+		// We explicitly filter the resulting events based on the requested events.
+		// In some cases, poll will report events we didn't ask for.
+		return RB_INT2NUM(events_from_poll_flags(arguments->waiting->result & arguments->flags));
+	} else {
 		return Qfalse;
 	}
-
-	// We explicitly filter the resulting events based on the requested events.
-	// In some cases, poll will report events we didn't ask for.
-	short flags = arguments->flags & NUM2INT(result);
-	
-	return INT2NUM(events_from_poll_flags(flags));
 };
 
 VALUE IO_Event_Selector_URing_io_wait(VALUE self, VALUE fiber, VALUE io, VALUE events) {
-	struct IO_Event_Selector_URing *data = NULL;
-	TypedData_Get_Struct(self, struct IO_Event_Selector_URing, &IO_Event_Selector_URing_Type, data);
+	struct IO_Event_Selector_URing *selector = NULL;
+	TypedData_Get_Struct(self, struct IO_Event_Selector_URing, &IO_Event_Selector_URing_Type, selector);
 	
 	int descriptor = IO_Event_Selector_io_descriptor(io);
-	struct io_uring_sqe *sqe = io_get_sqe(data);
+	struct io_uring_sqe *sqe = io_get_sqe(selector);
 	
 	short flags = poll_flags_from_events(NUM2INT(events));
 	
 	if (DEBUG) fprintf(stderr, "IO_Event_Selector_URing_io_wait:io_uring_prep_poll_add(descriptor=%d, flags=%d, fiber=%p)\n", descriptor, flags, (void*)fiber);
 	
 	io_uring_prep_poll_add(sqe, descriptor, flags);
-	io_uring_sqe_set_data(sqe, (void*)fiber);
+	
+	struct IO_Event_Selector_URing_Waiting waiting = {
+		.fiber = fiber,
+	};
+	
+	struct IO_Event_Selector_URing_Completion *completion = IO_Event_Selector_URing_Completion_acquire(selector, &waiting);
+	
+	io_uring_sqe_set_data(sqe, completion);
 	
 	// If we are going to wait, we assume that we are waiting for a while:
-	io_uring_submit_pending(data);
+	io_uring_submit_pending(selector);
 	
 	struct io_wait_arguments io_wait_arguments = {
-		.data = data,
-		.fiber = fiber,
+		.selector = selector,
+		.waiting = &waiting,
 		.flags = flags
 	};
 	
-	return rb_rescue(io_wait_transfer, (VALUE)&io_wait_arguments, io_wait_rescue, (VALUE)&io_wait_arguments);
+	return rb_ensure(io_wait_transfer, (VALUE)&io_wait_arguments, io_wait_ensure, (VALUE)&io_wait_arguments);
 }
 
 #ifdef HAVE_RUBY_IO_BUFFER_H
@@ -417,8 +567,8 @@ static inline off_t io_seekable(int descriptor)
 #pragma mark - IO#read
 
 struct io_read_arguments {
-	struct IO_Event_Selector_URing *data;
-	VALUE fiber;
+	struct IO_Event_Selector_URing *selector;
+	struct IO_Event_Selector_URing_Waiting *waiting;
 	int descriptor;
 	char *buffer;
 	size_t length;
@@ -428,58 +578,67 @@ static VALUE
 io_read_submit(VALUE _arguments)
 {
 	struct io_read_arguments *arguments = (struct io_read_arguments *)_arguments;
-	struct IO_Event_Selector_URing *data = arguments->data;
-	struct io_uring_sqe *sqe = io_get_sqe(data);
+	struct IO_Event_Selector_URing *selector = arguments->selector;
+	struct io_uring_sqe *sqe = io_get_sqe(selector);
 	
-	if (DEBUG) fprintf(stderr, "io_read_submit:io_uring_prep_read(fiber=%p, descriptor=%d, buffer=%p, length=%ld)\n", (void*)arguments->fiber, arguments->descriptor, arguments->buffer, arguments->length);
+	if (DEBUG) fprintf(stderr, "io_read_submit:io_uring_prep_read(fiber=%p, descriptor=%d, buffer=%p, length=%ld)\n", (void*)arguments->waiting, arguments->descriptor, arguments->buffer, arguments->length);
 	
 	io_uring_prep_read(sqe, arguments->descriptor, arguments->buffer, arguments->length, io_seekable(arguments->descriptor));
-	io_uring_sqe_set_data(sqe, (void*)arguments->fiber);
-	io_uring_submit_now(data);
+	io_uring_sqe_set_data(sqe, arguments->waiting->completion);
+	io_uring_submit_now(selector);
+	
+	IO_Event_Selector_fiber_transfer(selector->backend.loop, 0, NULL);
 	
-	return IO_Event_Selector_fiber_transfer(data->backend.loop, 0, NULL);
+	return RB_INT2NUM(arguments->waiting->result);
 }
 
 static VALUE
-io_read_cancel(VALUE _arguments, VALUE exception)
+io_read_ensure(VALUE _arguments)
 {
 	struct io_read_arguments *arguments = (struct io_read_arguments *)_arguments;
-	struct IO_Event_Selector_URing *data = arguments->data;
+	struct IO_Event_Selector_URing *selector = arguments->selector;
 	
-	struct io_uring_sqe *sqe = io_get_sqe(data);
+	struct io_uring_sqe *sqe = io_get_sqe(selector);
 	
-	if (DEBUG) fprintf(stderr, "io_read_cancel:io_uring_prep_cancel(fiber=%p)\n", (void*)arguments->fiber);
+	if (DEBUG) fprintf(stderr, "io_read_cancel:io_uring_prep_cancel(fiber=%p)\n", (void*)arguments->waiting);
+	
+	// If the operation has already completed, we don't need to cancel it:
+	if (!arguments->waiting->result) {
+		io_uring_prep_cancel(sqe, (void*)arguments->waiting, 0);
+		io_uring_sqe_set_data(sqe, NULL);
+		io_uring_submit_now(selector);
+	}
 	
-	io_uring_prep_cancel(sqe, (void*)arguments->fiber, 0);
-	io_uring_sqe_set_data(sqe, NULL);
-	io_uring_submit_now(data);
+	IO_Event_Selector_URing_Waiting_cancel(arguments->selector, arguments->waiting);
 	
-	rb_exc_raise(exception);
+	return Qnil;
 }
 
 static int
-io_read(struct IO_Event_Selector_URing *data, VALUE fiber, int descriptor, char *buffer, size_t length)
+io_read(struct IO_Event_Selector_URing *selector, VALUE fiber, int descriptor, char *buffer, size_t length)
 {
-	struct io_read_arguments io_read_arguments = {
-		.data = data,
+	struct IO_Event_Selector_URing_Waiting waiting = {
 		.fiber = fiber,
+	};
+	
+	IO_Event_Selector_URing_Completion_acquire(selector, &waiting);
+	
+	struct io_read_arguments io_read_arguments = {
+		.selector = selector,
+		.waiting = &waiting,
 		.descriptor = descriptor,
 		.buffer = buffer,
 		.length = length
 	};
 	
-	int result = RB_NUM2INT(
-		rb_rescue(io_read_submit, (VALUE)&io_read_arguments, io_read_cancel, (VALUE)&io_read_arguments)
+	return RB_NUM2INT(
+		rb_ensure(io_read_submit, (VALUE)&io_read_arguments, io_read_ensure, (VALUE)&io_read_arguments)
 	);
-	
-	if (DEBUG) fprintf(stderr, "io_read:IO_Event_Selector_fiber_transfer -> %d\n", result);
-	
-	return result;
 }
 
 VALUE IO_Event_Selector_URing_io_read(VALUE self, VALUE fiber, VALUE io, VALUE buffer, VALUE _length, VALUE _offset) {
-	struct IO_Event_Selector_URing *data = NULL;
-	TypedData_Get_Struct(self, struct IO_Event_Selector_URing, &IO_Event_Selector_URing_Type, data);
+	struct IO_Event_Selector_URing *selector = NULL;
+	TypedData_Get_Struct(self, struct IO_Event_Selector_URing, &IO_Event_Selector_URing_Type, selector);
 	
 	int descriptor = IO_Event_Selector_io_descriptor(io);
 	
@@ -489,14 +648,14 @@ VALUE IO_Event_Selector_URing_io_read(VALUE self, VALUE fiber, VALUE io, VALUE b
 	
 	size_t length = NUM2SIZET(_length);
 	size_t offset = NUM2SIZET(_offset);
+	size_t total = 0;
 	
-	while (true) {
-		size_t maximum_size = size - offset;
-		if (DEBUG_IO_READ) fprintf(stderr, "io_read(%d, +%ld, %ld)\n", descriptor, offset, maximum_size);
-		int result = io_read(data, fiber, descriptor, (char*)base+offset, maximum_size);
-		if (DEBUG_IO_READ) fprintf(stderr, "io_read(%d, +%ld, %ld) -> %d\n", descriptor, offset, maximum_size, result);
+	size_t maximum_size = size - offset;
+	while (maximum_size) {
+		int result = io_read(selector, fiber, descriptor, (char*)base+offset, maximum_size);
 		
 		if (result > 0) {
+			total += result;
 			offset += result;
 			if ((size_t)result >= length) break;
 			length -= result;
@@ -507,9 +666,11 @@ VALUE IO_Event_Selector_URing_io_read(VALUE self, VALUE fiber, VALUE io, VALUE b
 		} else {
 			return rb_fiber_scheduler_io_result(-1, -result);
 		}
+		
+		maximum_size = size - offset;
 	}
 	
-	return rb_fiber_scheduler_io_result(offset, 0);
+	return rb_fiber_scheduler_io_result(total, 0);
 }
 
 static VALUE IO_Event_Selector_URing_io_read_compatible(int argc, VALUE *argv, VALUE self)
@@ -528,8 +689,8 @@ static VALUE IO_Event_Selector_URing_io_read_compatible(int argc, VALUE *argv, V
 #pragma mark - IO#write
 
 struct io_write_arguments {
-	struct IO_Event_Selector_URing *data;
-	VALUE fiber;
+	struct IO_Event_Selector_URing *selector;
+	struct IO_Event_Selector_URing_Waiting *waiting;
 	int descriptor;
 	char *buffer;
 	size_t length;
@@ -539,59 +700,67 @@ static VALUE
 io_write_submit(VALUE _argument)
 {
 	struct io_write_arguments *arguments = (struct io_write_arguments*)_argument;
-	struct IO_Event_Selector_URing *data = arguments->data;
+	struct IO_Event_Selector_URing *selector = arguments->selector;
 	
-	struct io_uring_sqe *sqe = io_get_sqe(data);
+	struct io_uring_sqe *sqe = io_get_sqe(selector);
 	
-	if (DEBUG) fprintf(stderr, "io_write_submit:io_uring_prep_write(fiber=%p, descriptor=%d, buffer=%p, length=%ld)\n", (void*)arguments->fiber, arguments->descriptor, arguments->buffer, arguments->length);
+	if (DEBUG) fprintf(stderr, "io_write_submit:io_uring_prep_write(fiber=%p, descriptor=%d, buffer=%p, length=%ld)\n", (void*)arguments->waiting, arguments->descriptor, arguments->buffer, arguments->length);
 	
 	io_uring_prep_write(sqe, arguments->descriptor, arguments->buffer, arguments->length, io_seekable(arguments->descriptor));
-	io_uring_sqe_set_data(sqe, (void*)arguments->fiber);
-	io_uring_submit_pending(data);
+	io_uring_sqe_set_data(sqe, arguments->waiting->completion);
+	io_uring_submit_pending(selector);
+	
+	IO_Event_Selector_fiber_transfer(selector->backend.loop, 0, NULL);
 	
-	return IO_Event_Selector_fiber_transfer(data->backend.loop, 0, NULL);
+	return RB_INT2NUM(arguments->waiting->result);
 }
 
 static VALUE
-io_write_cancel(VALUE _argument, VALUE exception)
+io_write_ensure(VALUE _argument)
 {
 	struct io_write_arguments *arguments = (struct io_write_arguments*)_argument;
-	struct IO_Event_Selector_URing *data = arguments->data;
+	struct IO_Event_Selector_URing *selector = arguments->selector;
 	
-	struct io_uring_sqe *sqe = io_get_sqe(data);
+	struct io_uring_sqe *sqe = io_get_sqe(selector);
 	
-	if (DEBUG) fprintf(stderr, "io_wait_rescue:io_uring_prep_cancel(%p)\n", (void*)arguments->fiber);
+	if (DEBUG) fprintf(stderr, "io_wait_rescue:io_uring_prep_cancel(%p)\n", (void*)arguments->waiting);
 	
-	io_uring_prep_cancel(sqe, (void*)arguments->fiber, 0);
-	io_uring_sqe_set_data(sqe, NULL);
-	io_uring_submit_now(data);
+	if (!arguments->waiting->result) {
+		io_uring_prep_cancel(sqe, (void*)arguments->waiting, 0);
+		io_uring_sqe_set_data(sqe, NULL);
+		io_uring_submit_now(selector);
+	}
 	
-	rb_exc_raise(exception);
+	IO_Event_Selector_URing_Waiting_cancel(arguments->selector, arguments->waiting);
+	
+	return Qnil;
 }
 
 static int
-io_write(struct IO_Event_Selector_URing *data, VALUE fiber, int descriptor, char *buffer, size_t length)
+io_write(struct IO_Event_Selector_URing *selector, VALUE fiber, int descriptor, char *buffer, size_t length)
 {
-	struct io_write_arguments arguments = {
-		.data = data,
+	struct IO_Event_Selector_URing_Waiting waiting = {
 		.fiber = fiber,
+	};
+	
+	IO_Event_Selector_URing_Completion_acquire(selector, &waiting);
+	
+	struct io_write_arguments arguments = {
+		.selector = selector,
+		.waiting = &waiting,
 		.descriptor = descriptor,
 		.buffer = buffer,
 		.length = length,
 	};
 	
-	int result = RB_NUM2INT(
-		rb_rescue(io_write_submit, (VALUE)&arguments, io_write_cancel, (VALUE)&arguments)
+	return RB_NUM2INT(
+		rb_ensure(io_write_submit, (VALUE)&arguments, io_write_ensure, (VALUE)&arguments)
 	);
-	
-	if (DEBUG) fprintf(stderr, "io_write:IO_Event_Selector_fiber_transfer -> %d\n", result);
-	
-	return result;
 }
 
 VALUE IO_Event_Selector_URing_io_write(VALUE self, VALUE fiber, VALUE io, VALUE buffer, VALUE _length, VALUE _offset) {
-	struct IO_Event_Selector_URing *data = NULL;
-	TypedData_Get_Struct(self, struct IO_Event_Selector_URing, &IO_Event_Selector_URing_Type, data);
+	struct IO_Event_Selector_URing *selector = NULL;
+	TypedData_Get_Struct(self, struct IO_Event_Selector_URing, &IO_Event_Selector_URing_Type, selector);
 	
 	int descriptor = IO_Event_Selector_io_descriptor(io);
 	
@@ -601,16 +770,18 @@ VALUE IO_Event_Selector_URing_io_write(VALUE self, VALUE fiber, VALUE io, VALUE
 	
 	size_t length = NUM2SIZET(_length);
 	size_t offset = NUM2SIZET(_offset);
+	size_t total = 0;
 	
 	if (length > size) {
 		rb_raise(rb_eRuntimeError, "Length exceeds size of buffer!");
 	}
-	
-	while (true) {
-		size_t maximum_size = size - offset;
-		int result = io_write(data, fiber, descriptor, (char*)base+offset, maximum_size);
+
+	size_t maximum_size = size - offset;
+	while (maximum_size) {
+		int result = io_write(selector, fiber, descriptor, (char*)base+offset, maximum_size);
 		
 		if (result > 0) {
+			total += result;
 			offset += result;
 			if ((size_t)result >= length) break;
 			length -= result;
@@ -621,9 +792,11 @@ VALUE IO_Event_Selector_URing_io_write(VALUE self, VALUE fiber, VALUE io, VALUE
 		} else {
 			return rb_fiber_scheduler_io_result(-1, -result);
 		}
+		
+		maximum_size = size - offset;
 	}
 	
-	return rb_fiber_scheduler_io_result(offset, 0);
+	return rb_fiber_scheduler_io_result(total, 0);
 }
 
 static VALUE IO_Event_Selector_URing_io_write_compatible(int argc, VALUE *argv, VALUE self)
@@ -646,17 +819,17 @@ static VALUE IO_Event_Selector_URing_io_write_compatible(int argc, VALUE *argv,
 static const int ASYNC_CLOSE = 1;
 
 VALUE IO_Event_Selector_URing_io_close(VALUE self, VALUE io) {
-	struct IO_Event_Selector_URing *data = NULL;
-	TypedData_Get_Struct(self, struct IO_Event_Selector_URing, &IO_Event_Selector_URing_Type, data);
+	struct IO_Event_Selector_URing *selector = NULL;
+	TypedData_Get_Struct(self, struct IO_Event_Selector_URing, &IO_Event_Selector_URing_Type, selector);
 	
 	int descriptor = IO_Event_Selector_io_descriptor(io);
 
 	if (ASYNC_CLOSE) {
-		struct io_uring_sqe *sqe = io_get_sqe(data);
+		struct io_uring_sqe *sqe = io_get_sqe(selector);
 		
 		io_uring_prep_close(sqe, descriptor);
 		io_uring_sqe_set_data(sqe, NULL);
-		io_uring_submit_now(data);
+		io_uring_submit_now(selector);
 	} else {
 		close(descriptor);
 	}
@@ -699,7 +872,7 @@ int timeout_nonblocking(struct __kernel_timespec *timespec) {
 }
 
 struct select_arguments {
-	struct IO_Event_Selector_URing *data;
+	struct IO_Event_Selector_URing *selector;
 	
 	int result;
 	
@@ -712,18 +885,18 @@ void * select_internal(void *_arguments) {
 	struct select_arguments * arguments = (struct select_arguments *)_arguments;
 	struct io_uring_cqe *cqe = NULL;
 	
-	arguments->result = io_uring_wait_cqe_timeout(&arguments->data->ring, &cqe, arguments->timeout);
+	arguments->result = io_uring_wait_cqe_timeout(&arguments->selector->ring, &cqe, arguments->timeout);
 	
 	return NULL;
 }
 
 static
 int select_internal_without_gvl(struct select_arguments *arguments) {
-	io_uring_submit_flush(arguments->data);
+	io_uring_submit_flush(arguments->selector);
 	
-	arguments->data->blocked = 1;
+	arguments->selector->blocked = 1;
 	rb_thread_call_without_gvl(select_internal, (void *)arguments, RUBY_UBF_IO, 0);
-	arguments->data->blocked = 0;
+	arguments->selector->blocked = 0;
 	
 	if (arguments->result == -ETIME) {
 		arguments->result = 0;
@@ -740,7 +913,8 @@ int select_internal_without_gvl(struct select_arguments *arguments) {
 }
 
 static inline
-unsigned select_process_completions(struct io_uring *ring) {
+unsigned select_process_completions(struct IO_Event_Selector_URing *selector) {
+	struct io_uring *ring = &selector->ring;
 	unsigned completed = 0;
 	unsigned head;
 	struct io_uring_cqe *cqe;
@@ -749,87 +923,96 @@ unsigned select_process_completions(struct io_uring *ring) {
 		++completed;
 		
 		// If the operation was cancelled, or the operation has no user data (fiber):
-		if (cqe->res == -ECANCELED || cqe->user_data == 0 || cqe->user_data == LIBURING_UDATA_TIMEOUT) {
+		if (cqe->user_data == 0 || cqe->user_data == LIBURING_UDATA_TIMEOUT) {
 			io_uring_cq_advance(ring, 1);
 			continue;
 		}
 		
-		VALUE fiber = (VALUE)cqe->user_data;
-		VALUE result = RB_INT2NUM(cqe->res);
-		
 		if (DEBUG) fprintf(stderr, "cqe res=%d user_data=%p\n", cqe->res, (void*)cqe->user_data);
 		
+		struct IO_Event_Selector_URing_Completion *completion = (void*)cqe->user_data;
+		struct IO_Event_Selector_URing_Waiting *waiting = completion->waiting;
+		
+		if (waiting) {
+			waiting->result = cqe->res;
+			waiting->flags = cqe->flags;
+		}
+		
 		io_uring_cq_advance(ring, 1);
 		
-		IO_Event_Selector_fiber_transfer(fiber, 1, &result);
+		if (waiting && waiting->fiber) {
+			IO_Event_Selector_fiber_transfer(waiting->fiber, 0, NULL);
+		}
+		
+		IO_Event_Selector_URing_Completion_release(selector, completion);
 	}
 	
-	// io_uring_cq_advance(ring, completed);
-	
 	if (DEBUG && completed > 0) fprintf(stderr, "select_process_completions(completed=%d)\n", completed);
 	
 	return completed;
 }
 
 VALUE IO_Event_Selector_URing_select(VALUE self, VALUE duration) {
-	struct IO_Event_Selector_URing *data = NULL;
-	TypedData_Get_Struct(self, struct IO_Event_Selector_URing, &IO_Event_Selector_URing_Type, data);
+	struct IO_Event_Selector_URing *selector = NULL;
+	TypedData_Get_Struct(self, struct IO_Event_Selector_URing, &IO_Event_Selector_URing_Type, selector);
 	
 	// Flush any pending events:
-	io_uring_submit_flush(data);
+	io_uring_submit_flush(selector);
 	
-	int ready = IO_Event_Selector_queue_flush(&data->backend);
+	int ready = IO_Event_Selector_queue_flush(&selector->backend);
 	
-	int result = select_process_completions(&data->ring);
+	int result = select_process_completions(selector);
 	
 	// If we:
 	// 1. Didn't process any ready fibers, and
 	// 2. Didn't process any events from non-blocking select (above), and
 	// 3. There are no items in the ready list,
 	// then we can perform a blocking select.
-	if (!ready && !result && !data->backend.ready) {
+	if (!ready && !result && !selector->backend.ready) {
 		// We might need to wait for events:
 		struct select_arguments arguments = {
-			.data = data,
+			.selector = selector,
 			.timeout = NULL,
 		};
 		
 		arguments.timeout = make_timeout(duration, &arguments.storage);
 		
-		if (!data->backend.ready && !timeout_nonblocking(arguments.timeout)) {
+		if (!selector->backend.ready && !timeout_nonblocking(arguments.timeout)) {
 			// This is a blocking operation, we wait for events:
 			result = select_internal_without_gvl(&arguments);
+			
+			// After waiting/flushing the SQ, check if there are any completions:
+			if (result > 0) {
+				result = select_process_completions(selector);
+			}
 		}
-		
-		// After waiting/flushing the SQ, check if there are any completions:
-		result = select_process_completions(&data->ring);
 	}
 	
 	return RB_INT2NUM(result);
 }
 
 VALUE IO_Event_Selector_URing_wakeup(VALUE self) {
-	struct IO_Event_Selector_URing *data = NULL;
-	TypedData_Get_Struct(self, struct IO_Event_Selector_URing, &IO_Event_Selector_URing_Type, data);
+	struct IO_Event_Selector_URing *selector = NULL;
+	TypedData_Get_Struct(self, struct IO_Event_Selector_URing, &IO_Event_Selector_URing_Type, selector);
 	
 	// If we are blocking, we can schedule a nop event to wake up the selector:
-	if (data->blocked) {
+	if (selector->blocked) {
 		struct io_uring_sqe *sqe = NULL;
 		
 		while (true) {
-			sqe = io_uring_get_sqe(&data->ring);
+			sqe = io_uring_get_sqe(&selector->ring);
 			if (sqe) break;
 			
 			rb_thread_schedule();
 			
 			// It's possible we became unblocked already, so we can assume the selector has already cycled at least once:
-			if (!data->blocked) return Qfalse;
+			if (!selector->blocked) return Qfalse;
 		}
 		
 		io_uring_prep_nop(sqe);
-		// If you don't set this line, the SQE will eventually be recycled and have valid user data which can cause odd behaviour:
+		// If you don't set this line, the SQE will eventually be recycled and have valid user selector which can cause odd behaviour:
 		io_uring_sqe_set_data(sqe, NULL);
-		io_uring_submit(&data->ring);
+		io_uring_submit(&selector->ring);
 		
 		return Qtrue;
 	}