uringmachine 0.25.0 → 0.27.0

data/ext/um/um.c

@@ -60,8 +60,8 @@ inline void um_teardown(struct um *machine) {
 }
 
 inline struct io_uring_sqe *um_get_sqe(struct um *machine, struct um_op *op) {
-  DEBUG_PRINTF("-> %p um_get_sqe: op %p kind=%s unsubmitted=%d pending=%d total=%lu\n",
-    &machine->ring, op, um_op_kind_name(op ? op->kind : OP_UNDEFINED),
+  DEBUG_PRINTF("* um_get_sqe: op %p kind=%s ref_count=%d flags=%x unsubmitted=%d pending=%d total=%lu\n",
+    op, um_op_kind_name(op ? op->kind : OP_UNDEFINED), op ? op->ref_count : 0, op ? op->flags : 0,
     machine->metrics.ops_unsubmitted, machine->metrics.ops_pending, machine->metrics.total_ops
   );
 
@@ -112,12 +112,11 @@ void *um_submit_without_gvl(void *ptr) {
 }
 
 inline uint um_submit(struct um *machine) {
-  DEBUG_PRINTF("-> %p um_submit: unsubmitted=%d pending=%d total=%lu\n",
-    &machine->ring, machine->metrics.ops_unsubmitted, machine->metrics.ops_pending,
-    machine->metrics.total_ops
+  DEBUG_PRINTF("> um_submit: unsubmitted=%d pending=%d total=%lu\n",
+    machine->metrics.ops_unsubmitted, machine->metrics.ops_pending, machine->metrics.total_ops
   );
   if (!machine->metrics.ops_unsubmitted) {
-    DEBUG_PRINTF("<- %p um_submit: no unsubmitted SQEs, early return\n", &machine->ring);
+    DEBUG_PRINTF("< %p um_submit: no unsubmitted SQEs, early return\n", &machine->ring);
     return 0;
   }
 
@@ -127,7 +126,7 @@ inline uint um_submit(struct um *machine) {
   else
     ctx.result = io_uring_submit(&machine->ring);
 
-  DEBUG_PRINTF("<- %p um_submit: result=%d\n", &machine->ring, ctx.result);
+  DEBUG_PRINTF("< um_submit: result=%d\n", ctx.result);
 
   if (ctx.result < 0)
     rb_syserr_fail(-ctx.result, strerror(-ctx.result));
@@ -136,53 +135,76 @@ inline uint um_submit(struct um *machine) {
   return ctx.result;
 }
 
+static inline void um_schedule_op(struct um *machine, struct um_op *op) {
+  op->flags |= OP_F_SCHEDULED;
+  um_runqueue_push(machine, op);
+  op->ref_count++;
+}
+
 static inline void um_process_cqe(struct um *machine, struct io_uring_cqe *cqe) {
   struct um_op *op = (struct um_op *)cqe->user_data;
   if (DEBUG) {
     if (op) {
-      DEBUG_PRINTF("<- %p um_process_cqe: op %p kind %s flags %d cqe_res %d cqe_flags %d pending %d\n",
-        &machine->ring, op, um_op_kind_name(op->kind), op->flags, cqe->res, cqe->flags, machine->metrics.ops_pending
+      DEBUG_PRINTF("* um_process_cqe: op=%p kind=%s ref_count=%d flags=%x cqe_res=%d cqe_flags=%x pending=%d\n",
+        op, um_op_kind_name(op->kind), op->ref_count, op->flags,
+        cqe->res, cqe->flags, machine->metrics.ops_pending
       );
     }
     else {
-      DEBUG_PRINTF("<- %p um_process_cqe: op NULL cqe_res %d cqe_flags %d pending %d\n",
-        &machine->ring, cqe->res, cqe->flags, machine->metrics.ops_pending
+      DEBUG_PRINTF("* um_process_cqe: op=NULL cqe_res=%d cqe_flags=%x pending=%d\n",
+        cqe->res, cqe->flags, machine->metrics.ops_pending
       );
     }
   }
   if (unlikely(!op)) return;
 
-  if (!(cqe->flags & IORING_CQE_F_MORE))
-    machine->metrics.ops_pending--;
+  // A multishot operation is still in progress if CQE has the F_MORE flag set
+  int done = OP_MULTISHOT_P(op) ? !(cqe->flags & IORING_CQE_F_MORE) : true;
 
-  if (op->flags & OP_F_FREE_ON_COMPLETE) {
-    if (op->flags & OP_F_TRANSIENT)
-      um_op_transient_remove(machine, op);
-
-    um_op_free(machine, op);
+  // F_TRANSIENT means the operation was put on the transient list. Transient
+  // ops are usually async ops where the app doesn't care when they are done or
+  // how. We hold on to those ops on the transient list, where we can mark the
+  // corresponding buffer during a GC.
+  if (OP_TRANSIENT_P(op)) {
+    machine->metrics.ops_pending--;
+    um_op_transient_remove(machine, op);
+    if (op->ref_count > 1) {
+      op->result.res    = cqe->res;
+      op->result.flags  = cqe->flags;
+      op->flags |= OP_F_CQE_SEEN | OP_F_CQE_DONE;
+    }
+    um_op_release(machine, op);
     return;
   }
 
-  op->flags |= OP_F_COMPLETED;
-  if (unlikely((cqe->res == -ECANCELED) && (op->flags & OP_F_IGNORE_CANCELED))) return;
-  if (unlikely(op->flags & OP_F_CANCELED)) return;
+  if (done) {
+    machine->metrics.ops_pending--;
+    um_op_release(machine, op);
+  }
 
-  if (op->flags & OP_F_TRANSIENT)
-    um_op_transient_remove(machine, op);
+  if (unlikely(OP_CANCELED_P(op))) {
+    // multishot ops may generate multiple CQEs, we release only on the last
+    // one for the op.
+    if (done) {
+      op->flags |= OP_F_CQE_SEEN | OP_F_CQE_DONE;
+      um_op_release(machine, op);
+    }
+    return;
+  }
 
-  if (op->flags & OP_F_MULTISHOT) {
+  if (OP_MULTISHOT_P(op)) {
     um_op_multishot_results_push(machine, op, cqe->res, cqe->flags);
-    if (op->multishot_result_count > 1)
-      return;
+
+    op->flags |= done ? (OP_F_CQE_SEEN | OP_F_CQE_DONE) : OP_F_CQE_SEEN;
+    if (!OP_SCHEDULED_P(op)) um_schedule_op(machine, op);
   }
   else {
+    // single shot
     op->result.res    = cqe->res;
     op->result.flags  = cqe->flags;
+    op->flags |= OP_F_CQE_SEEN | OP_F_CQE_DONE;
+    if (!OP_ASYNC_P(op)) um_schedule_op(machine, op);
   }
-
-  if (op->flags & OP_F_ASYNC) return;
-
-  um_runqueue_push(machine, op);
 }
 
 // copied from liburing/queue.c
@@ -191,8 +213,8 @@ static inline int cq_ring_needs_flush(struct io_uring *ring) {
 }
 
 static inline int um_process_ready_cqes(struct um *machine) {
-  DEBUG_PRINTF("-> %p um_process_ready_cqes: unsubmitted=%d pending=%d total=%lu\n",
-    &machine->ring, machine->metrics.ops_unsubmitted, machine->metrics.ops_pending, machine->metrics.total_ops
+  DEBUG_PRINTF("> um_process_ready_cqes: unsubmitted=%d pending=%d total=%lu\n",
+    machine->metrics.ops_unsubmitted, machine->metrics.ops_pending, machine->metrics.total_ops
   );
 
   unsigned total_count = 0;
@@ -211,9 +233,9 @@ iterate:
   if (overflow_checked) goto done;
 
   if (cq_ring_needs_flush(&machine->ring)) {
-    DEBUG_PRINTF("-> %p io_uring_enter\n", &machine->ring);
+    DEBUG_PRINTF("> io_uring_enter\n");
     int ret = io_uring_enter(machine->ring.ring_fd, 0, 0, IORING_ENTER_GETEVENTS, NULL);
-    DEBUG_PRINTF("<- %p io_uring_enter: result=%d\n", &machine->ring, ret);
+    DEBUG_PRINTF("< io_uring_enter: result=%d\n", ret);
     if (ret < 0)
       rb_syserr_fail(-ret, strerror(-ret));
 
@@ -222,7 +244,7 @@ iterate:
   }
 
 done:
-  DEBUG_PRINTF("<- %p um_process_ready_cqes: total_processed=%u\n", &machine->ring, total_count);
+  DEBUG_PRINTF("< um_process_ready_cqes: total_processed=%u\n", total_count);
 
   return total_count;
 }
@@ -237,8 +259,8 @@ struct wait_for_cqe_ctx {
 void *um_wait_for_cqe_without_gvl(void *ptr) {
   struct wait_for_cqe_ctx *ctx = ptr;
   if (ctx->machine->metrics.ops_unsubmitted) {
-    DEBUG_PRINTF("-> %p io_uring_submit_and_wait_timeout: unsubmitted=%d pending=%d total=%lu\n",
-      &ctx->machine->ring, ctx->machine->metrics.ops_unsubmitted, ctx->machine->metrics.ops_pending,
+    DEBUG_PRINTF("> io_uring_submit_and_wait_timeout: unsubmitted=%d pending=%d total=%lu\n",
+      ctx->machine->metrics.ops_unsubmitted, ctx->machine->metrics.ops_pending,
       ctx->machine->metrics.total_ops
     );
 
@@ -249,16 +271,16 @@ void *um_wait_for_cqe_without_gvl(void *ptr) {
     // https://github.com/axboe/liburing/issues/1280
     int ret = io_uring_submit_and_wait_timeout(&ctx->machine->ring, &ctx->cqe, ctx->wait_nr, NULL, NULL);
     ctx->machine->metrics.ops_unsubmitted = 0;
-    DEBUG_PRINTF("<- %p io_uring_submit_and_wait_timeout: result=%d\n", &ctx->machine->ring, ret);
+    DEBUG_PRINTF("< io_uring_submit_and_wait_timeout: result=%d\n", ret);
     ctx->result = (ret > 0 && !ctx->cqe) ? -EINTR : ret;
   }
   else {
-    DEBUG_PRINTF("-> %p io_uring_wait_cqes: unsubmitted=%d pending=%d total=%lu\n",
-      &ctx->machine->ring, ctx->machine->metrics.ops_unsubmitted, ctx->machine->metrics.ops_pending,
+    DEBUG_PRINTF("> io_uring_wait_cqes: unsubmitted=%d pending=%d total=%lu\n",
+      ctx->machine->metrics.ops_unsubmitted, ctx->machine->metrics.ops_pending,
       ctx->machine->metrics.total_ops
     );
     ctx->result = io_uring_wait_cqes(&ctx->machine->ring, &ctx->cqe, ctx->wait_nr, NULL, NULL);
-    DEBUG_PRINTF("<- %p io_uring_wait_cqes: result=%d\n", &ctx->machine->ring, ctx->result);
+    DEBUG_PRINTF("< io_uring_wait_cqes: result=%d\n", ctx->result);
   }
   return NULL;
 }
@@ -290,6 +312,7 @@ inline void *um_wait_for_sidecar_signal(void *ptr) {
 // either 1 - where we wait for at least one CQE to be ready, or 0, where we
 // don't wait, and just process any CQEs that already ready.
 static inline void um_wait_for_and_process_ready_cqes(struct um *machine, int wait_nr) {
+  DEBUG_PRINTF("* um_wait_for_and_process_ready_cqes wait_nr=%d\n", wait_nr);
   struct wait_for_cqe_ctx ctx = { .machine = machine, .cqe = NULL, .wait_nr = wait_nr };
   machine->metrics.total_waits++;
 
@@ -347,14 +370,16 @@ inline void um_profile_switch(struct um *machine, VALUE next_fiber) {
 }
 
 inline VALUE process_runqueue_op(struct um *machine, struct um_op *op) {
-  DEBUG_PRINTF("-> %p process_runqueue_op: op %p\n", &machine->ring, op);
+  DEBUG_PRINTF("* process_runqueue_op: op=%p kind=%s ref_count=%d flags=%x\n",
+    op, um_op_kind_name(op->kind), op->ref_count, op->flags
+  );
 
   machine->metrics.total_switches++;
   VALUE fiber = op->fiber;
   VALUE value = op->value;
 
-  if (unlikely(op->flags & OP_F_TRANSIENT))
-    um_op_free(machine, op);
+  op->flags &= ~OP_F_SCHEDULED;
+  um_op_release(machine, op);
 
   if (machine->profile_mode) um_profile_switch(machine, fiber);
   VALUE ret = rb_fiber_transfer(fiber, 1, &value);
@@ -364,15 +389,19 @@ inline VALUE process_runqueue_op(struct um *machine, struct um_op *op) {
 }
 
 inline VALUE um_switch(struct um *machine) {
-  DEBUG_PRINTF("-> %p um_switch: unsubmitted=%d pending=%d total=%lu\n",
-    &machine->ring, machine->metrics.ops_unsubmitted, machine->metrics.ops_pending,
+  DEBUG_PRINTF("* um_switch: unsubmitted=%d pending=%d total=%lu\n",
+    machine->metrics.ops_unsubmitted, machine->metrics.ops_pending,
     machine->metrics.total_ops
   );
 
   while (true) {
     struct um_op *op = um_runqueue_shift(machine);
     if (op) {
-      if (unlikely(op->flags & OP_F_RUNQUEUE_SKIP)) continue;
+      if (unlikely(OP_SKIP_P(op))) {
+        op->flags &= ~OP_F_SCHEDULED;
+        um_op_release(machine, op);
+        continue;
+      }
 
       // in test mode we want to process I/O on each snooze
       if (unlikely(machine->test_mode && (op->kind == OP_SCHEDULE))) {
@@ -393,29 +422,62 @@ inline VALUE um_yield(struct um *machine) {
   return ret;
 }
 
-void um_cancel_op(struct um *machine, struct um_op *op) {
+inline void um_cancel_op(struct um *machine, struct um_op *op) {
   struct io_uring_sqe *sqe = um_get_sqe(machine, NULL);
-  io_uring_prep_cancel64(sqe, (long long)op, 0);
+  io_uring_prep_cancel(sqe, op, IORING_ASYNC_CANCEL_USERDATA);
+  sqe->flags = IOSQE_CQE_SKIP_SUCCESS;
 }
 
-inline void um_cancel_and_wait(struct um *machine, struct um_op *op) {
+inline void um_cancel_op_and_discard_cqe(struct um *machine, struct um_op *op) {
+  DEBUG_PRINTF("* um_cancel_op_and_discard_cqe op=%p kind=%s ref_count=%d flags=%x\n",
+    op, um_op_kind_name(op->kind), op->ref_count, op->flags
+  );
+  um_cancel_op(machine, op);
+  op->flags |= OP_F_CANCELED;
+}
+
+inline void um_cancel_op_and_await_cqe(struct um *machine, struct um_op *op) {
+  DEBUG_PRINTF("* um_cancel_op_and_await_cqe op=%p kind=%s ref_count=%d flags=%x\n",
+    op, um_op_kind_name(op->kind), op->ref_count, op->flags
+  );
   um_cancel_op(machine, op);
 
   VALUE fiber = rb_fiber_current();
   rb_set_add(machine->pending_fibers, fiber);
-  while (!um_op_completed_p(op)) {
+  int multishot_wait_count = 0;
+  while (!OP_CQE_DONE_P(op)) {
+    if (OP_MULTISHOT_P(op)) {
+      // I noticed that with multishot timeout ops, there seems to be once in a
+      // while a race condition where the cancel would not register, causing
+      // this function to block forever, waiting for the operation to be done.
+      // The following mechanism reissues a cancel every 4 iterations, which
+      // seems to fix the problem. Not clear if this is a bug in io_uring.
+      multishot_wait_count++;
+      if (!(multishot_wait_count % 4)) um_cancel_op(machine, op);
+      um_op_multishot_results_clear(machine, op);
+      op->flags &= ~OP_F_CQE_SEEN;
+    }
     um_switch(machine);
   }
   rb_set_delete(machine->pending_fibers, fiber);
 }
 
-inline int um_check_completion(struct um *machine, struct um_op *op) {
-  if (!um_op_completed_p(op)) {
-    um_cancel_and_wait(machine, op);
+int um_verify_op_completion(struct um *machine, struct um_op *op, int await_cancelled) {
+  if (unlikely(!OP_CQE_DONE_P(op))) {
+    if (await_cancelled)
+      um_cancel_op_and_await_cqe(machine, op);
+    else
+      um_cancel_op_and_discard_cqe(machine, op);
     return 0;
   }
 
-  um_raise_on_error_result(op->result.res);
+  int res = op->result.res;
+
+  // on error we release the op and immediately raise an exception
+  if (unlikely(res < 0 && res != -ETIME)) {
+    um_op_release(machine, op);
+    um_raise_on_error_result(res);
+  }
   return 1;
 }
 
@@ -426,25 +488,31 @@ VALUE um_wakeup(struct um *machine) {
   return Qnil;
 }
 
-inline void um_prep_op(struct um *machine, struct um_op *op, enum um_op_kind kind, unsigned flags) {
+inline void um_prep_op(struct um *machine, struct um_op *op, enum um_op_kind kind, uint ref_count, unsigned flags) {
   memset(op, 0, sizeof(struct um_op));
   op->kind = kind;
+  op->ref_count = ref_count;
   op->flags = flags;
 
-  VALUE fiber = (flags & OP_F_FREE_ON_COMPLETE) ? Qnil : rb_fiber_current();
+  VALUE fiber = OP_ASYNC_P(op) ? Qnil : rb_fiber_current();
+  if (OP_TRANSIENT_P(op)) um_op_transient_add(machine, op);
+
   RB_OBJ_WRITE(machine->self, &op->fiber, fiber);
   RB_OBJ_WRITE(machine->self, &op->value, Qnil);
   RB_OBJ_WRITE(machine->self, &op->async_op, Qnil);
 }
 
 inline void um_schedule(struct um *machine, VALUE fiber, VALUE value) {
-  struct um_op *op = um_op_alloc(machine);
+  struct um_op *op = um_op_acquire(machine);
   memset(op, 0, sizeof(struct um_op));
   op->kind = OP_SCHEDULE;
-  op->flags = OP_F_TRANSIENT;
+  op->ref_count = 1;
+  op->flags = OP_F_SCHEDULED;
+
   RB_OBJ_WRITE(machine->self, &op->fiber, fiber);
   RB_OBJ_WRITE(machine->self, &op->value, value);
   RB_OBJ_WRITE(machine->self, &op->async_op, Qnil);
+
   um_runqueue_push(machine, op);
 }
 
@@ -457,18 +525,14 @@ struct op_ctx {
   struct um_queue *queue;
   void *read_buf;
   int read_maxlen;
-  struct __kernel_timespec ts;
   int flags;
 };
 
 VALUE um_timeout_complete(VALUE arg) {
   struct op_ctx *ctx = (struct op_ctx *)arg;
 
-  if (!um_op_completed_p(ctx->op)) {
-    um_cancel_op(ctx->machine, ctx->op);
-    ctx->op->flags |= OP_F_TRANSIENT | OP_F_IGNORE_CANCELED;
-    um_op_transient_add(ctx->machine, ctx->op);
-  }
+  if (!OP_CQE_DONE_P(ctx->op)) um_cancel_op_and_discard_cqe(ctx->machine, ctx->op);
+  um_op_release(ctx->machine, ctx->op);
 
   return Qnil;
 }
@@ -477,8 +541,8 @@ VALUE um_timeout(struct um *machine, VALUE interval, VALUE class) {
   static ID ID_new = 0;
   if (!ID_new) ID_new = rb_intern("new");
 
-  struct um_op *op = um_op_alloc(machine);
-  um_prep_op(machine, op, OP_TIMEOUT, 0);
+  struct um_op *op = um_op_acquire(machine);
+  um_prep_op(machine, op, OP_TIMEOUT, 2, 0);
   op->ts = um_double_to_timespec(NUM2DBL(interval));
   RB_OBJ_WRITE(machine->self, &op->fiber, rb_fiber_current());
   RB_OBJ_WRITE(machine->self, &op->value, rb_funcall(class, ID_new, 0));
@@ -500,20 +564,20 @@ VALUE um_timeout(struct um *machine, VALUE interval, VALUE class) {
 VALUE um_sleep(struct um *machine, double duration) {
   if (duration <= 0) duration = SLEEP_FOREVER_DURATION;
 
-  struct um_op op;
-  um_prep_op(machine, &op, OP_SLEEP, 0);
-  op.ts = um_double_to_timespec(duration);
-  struct io_uring_sqe *sqe = um_get_sqe(machine, &op);
-  io_uring_prep_timeout(sqe, &op.ts, 0, 0);
+  struct um_op *op = um_op_acquire(machine);
+  um_prep_op(machine, op, OP_SLEEP, 2, 0);
+  op->ts = um_double_to_timespec(duration);
+  struct io_uring_sqe *sqe = um_get_sqe(machine, op);
+  io_uring_prep_timeout(sqe, &op->ts, 0, 0);
 
   VALUE ret = um_yield(machine);
 
-  if (!um_op_completed_p(&op))
-    um_cancel_and_wait(machine, &op);
-  else {
-    if (op.result.res != -ETIME) um_raise_on_error_result(op.result.res);
-    ret = DBL2NUM(duration);
-  }
+  DEBUG_PRINTF("sleep resume op %p ref_count %d flags: %x\n",
+    op, op->ref_count, op->flags
+  );
+
+  if (likely(um_verify_op_completion(machine, op, false))) ret = DBL2NUM(duration);
+  um_op_release(machine, op);
 
   RAISE_IF_EXCEPTION(ret);
   RB_GC_GUARD(ret);
@@ -521,19 +585,20 @@ VALUE um_sleep(struct um *machine, double duration) {
 }
 
 VALUE um_read(struct um *machine, int fd, VALUE buffer, size_t maxlen, ssize_t buffer_offset, __u64 file_offset) {
-  struct um_op op;
-  um_prep_op(machine, &op, OP_READ, 0);
-  struct io_uring_sqe *sqe = um_get_sqe(machine, &op);
   void *ptr = um_prepare_read_buffer(buffer, maxlen, buffer_offset);
+
+  struct um_op *op = um_op_acquire(machine);
+  um_prep_op(machine, op, OP_READ, 2, 0);
+  struct io_uring_sqe *sqe = um_get_sqe(machine, op);
   io_uring_prep_read(sqe, fd, ptr, maxlen, file_offset);
 
   VALUE ret = um_yield(machine);
 
-  if (um_check_completion(machine, &op)) {
-    um_update_read_buffer(buffer, buffer_offset, op.result.res);
-    ret = INT2NUM(op.result.res);
-
+  if (likely(um_verify_op_completion(machine, op, true))) {
+    um_update_read_buffer(buffer, buffer_offset, op->result.res);
+    ret = INT2NUM(op->result.res);
   }
+  um_op_release(machine, op);
 
   RAISE_IF_EXCEPTION(ret);
   RB_GC_GUARD(ret);
@@ -541,39 +606,38 @@ VALUE um_read(struct um *machine, int fd, VALUE buffer, size_t maxlen, ssize_t b
 }
 
 size_t um_read_raw(struct um *machine, int fd, char *buffer, size_t maxlen) {
-  struct um_op op;
-  um_prep_op(machine, &op, OP_READ, 0);
-  struct io_uring_sqe *sqe = um_get_sqe(machine, &op);
+  struct um_op *op = um_op_acquire(machine);
+  um_prep_op(machine, op, OP_READ, 2, 0);
+  struct io_uring_sqe *sqe = um_get_sqe(machine, op);
   io_uring_prep_read(sqe, fd, buffer, maxlen, -1);
 
+  int res = 0;
   VALUE ret = um_yield(machine);
 
-  if (um_check_completion(machine, &op)) {
-    return op.result.res;
-  }
+  if (likely(um_verify_op_completion(machine, op, true))) res = op->result.res;
+  um_op_release(machine, op);
 
   RAISE_IF_EXCEPTION(ret);
   RB_GC_GUARD(ret);
-  return 0;
+  return res;
 }
 
 VALUE um_write(struct um *machine, int fd, VALUE buffer, size_t len, __u64 file_offset) {
   const void *base;
   size_t size;
-  um_get_buffer_bytes_for_writing(buffer, &base, &size);
+  um_get_buffer_bytes_for_writing(buffer, &base, &size, true);
   if ((len == (size_t)-1) || (len > size)) len = size;
   if (unlikely(!len)) return INT2NUM(0);
 
-  struct um_op op;
-  um_prep_op(machine, &op, OP_WRITE, 0);
-  struct io_uring_sqe *sqe = um_get_sqe(machine, &op);
-
+  struct um_op *op = um_op_acquire(machine);
+  um_prep_op(machine, op, OP_WRITE, 2, 0);
+  struct io_uring_sqe *sqe = um_get_sqe(machine, op);
   io_uring_prep_write(sqe, fd, base, len, file_offset);
 
   VALUE ret = um_yield(machine);
 
-  if (um_check_completion(machine, &op))
-    ret = INT2NUM(op.result.res);
+  if (likely(um_verify_op_completion(machine, op, true))) ret = INT2NUM(op->result.res);
+  um_op_release(machine, op);
 
   RAISE_IF_EXCEPTION(ret);
   RB_GC_GUARD(ret);
@@ -581,19 +645,20 @@ VALUE um_write(struct um *machine, int fd, VALUE buffer, size_t len, __u64 file_
 }
 
 size_t um_write_raw(struct um *machine, int fd, const char *buffer, size_t maxlen) {
-  struct um_op op;
-  um_prep_op(machine, &op, OP_WRITE, 0);
-  struct io_uring_sqe *sqe = um_get_sqe(machine, &op);
+  struct um_op *op = um_op_acquire(machine);
+  um_prep_op(machine, op, OP_WRITE, 2, 0);
+  struct io_uring_sqe *sqe = um_get_sqe(machine, op);
   io_uring_prep_write(sqe, fd, buffer, maxlen, 0);
 
+  int res = 0;
   VALUE ret = um_yield(machine);
 
-  if (um_check_completion(machine, &op))
-    return op.result.res;
+  if (likely(um_verify_op_completion(machine, op, true))) res = op->result.res;
+  um_op_release(machine, op);
 
   RAISE_IF_EXCEPTION(ret);
   RB_GC_GUARD(ret);
-  return 0;
+  return res;
 }
 
 VALUE um_writev(struct um *machine, int fd, int argc, VALUE *argv) {
@@ -607,7 +672,6 @@ VALUE um_writev(struct um *machine, int fd, int argc, VALUE *argv) {
   struct iovec *iovecs = um_alloc_iovecs_for_writing(argc, argv, &total_len);
   struct iovec *iovecs_ptr = iovecs;
   int iovecs_len = argc;
-  struct um_op op;
   VALUE ret = Qnil;
   int writev_res = 0;
 
@@ -615,31 +679,34 @@ VALUE um_writev(struct um *machine, int fd, int argc, VALUE *argv) {
     free(iovecs);
     return INT2NUM(0);
   }
+
+  struct um_op *op = um_op_acquire(machine);
   len = total_len;
-  while (len) {
-    um_prep_op(machine, &op, OP_WRITEV, 0);
-    struct io_uring_sqe *sqe = um_get_sqe(machine, &op);
+  while (true) {
+    op->iovecs = iovecs;
+    um_prep_op(machine, op, OP_WRITEV, 2, OP_F_FREE_IOVECS);
+    struct io_uring_sqe *sqe = um_get_sqe(machine, op);
     io_uring_prep_writev(sqe, fd, iovecs_ptr, iovecs_len, file_offset);
 
     ret = um_yield(machine);
 
-    int completed = um_op_completed_p(&op);
-    if (unlikely(!completed)) goto cancelled;
+    if (unlikely(!OP_CQE_DONE_P(op))) goto cancelled;
 
-    writev_res = op.result.res;
+    writev_res = op->result.res;
     if (unlikely(writev_res < 0)) goto done;
 
     len -= writev_res;
-    if (len) {
-      um_advance_iovecs_for_writing(&iovecs_ptr, &iovecs_len, (size_t)writev_res);
-      if (file_offset != (__u64)-1) file_offset += writev_res;
-    }
+    if (!len) goto done;
+
+    um_advance_iovecs_for_writing(&iovecs_ptr, &iovecs_len, (size_t)writev_res);
+    if (file_offset != (__u64)-1) file_offset += writev_res;
   }
 
 cancelled:
-  um_cancel_and_wait(machine, &op);
+  um_cancel_op_and_await_cqe(machine, op);
 done:
-  free(iovecs);
+  um_op_release(machine, op);
+
   RAISE_IF_EXCEPTION(ret);
   RB_GC_GUARD(ret);
   um_raise_on_error_result(writev_res);
@@ -649,31 +716,29 @@ done:
 VALUE um_write_async(struct um *machine, int fd, VALUE buffer, size_t len, __u64 file_offset) {
   const void *base;
   size_t size;
-  um_get_buffer_bytes_for_writing(buffer, &base, &size);
+  um_get_buffer_bytes_for_writing(buffer, &base, &size, true);
   if ((len == (size_t)-1) || (len > size)) len = size;
   if (unlikely(!len)) return INT2NUM(0);
 
-  struct um_op *op = um_op_alloc(machine);
-  um_prep_op(machine, op, OP_WRITE_ASYNC, OP_F_TRANSIENT | OP_F_FREE_ON_COMPLETE);
+  struct um_op *op = um_op_acquire(machine);
+  um_prep_op(machine, op, OP_WRITE_ASYNC, 1, OP_F_ASYNC | OP_F_TRANSIENT);
   RB_OBJ_WRITE(machine->self, &op->value, buffer);
-
   struct io_uring_sqe *sqe = um_get_sqe(machine, op);
   io_uring_prep_write(sqe, fd, base, len, file_offset);
-  um_op_transient_add(machine, op);
 
   return buffer;
 }
 
 VALUE um_close(struct um *machine, int fd) {
-  struct um_op op;
-  um_prep_op(machine, &op, OP_CLOSE, 0);
-  struct io_uring_sqe *sqe = um_get_sqe(machine, &op);
+  struct um_op *op = um_op_acquire(machine);
+  um_prep_op(machine, op, OP_CLOSE, 2, 0);
+  struct io_uring_sqe *sqe = um_get_sqe(machine, op);
   io_uring_prep_close(sqe, fd);
 
   VALUE ret = um_yield(machine);
 
-  if (um_check_completion(machine, &op))
-    ret = INT2NUM(fd);
+  if (likely(um_verify_op_completion(machine, op, false))) ret = INT2NUM(fd);
+  um_op_release(machine, op);
 
   RAISE_IF_EXCEPTION(ret);
   RB_GC_GUARD(ret);
@@ -681,9 +746,8 @@ VALUE um_close(struct um *machine, int fd) {
 }
 
 VALUE um_close_async(struct um *machine, int fd) {
-  struct um_op *op = um_op_alloc(machine);
-  um_prep_op(machine, op, OP_CLOSE_ASYNC, OP_F_FREE_ON_COMPLETE);
-
+  struct um_op *op = um_op_acquire(machine);
+  um_prep_op(machine, op, OP_CLOSE_ASYNC, 1, OP_F_ASYNC);
   struct io_uring_sqe *sqe = um_get_sqe(machine, op);
   io_uring_prep_close(sqe, fd);
 
@@ -691,15 +755,15 @@ VALUE um_close_async(struct um *machine, int fd) {
 }
 
 VALUE um_accept(struct um *machine, int fd) {
-  struct um_op op;
-  um_prep_op(machine, &op, OP_ACCEPT, 0);
-  struct io_uring_sqe *sqe = um_get_sqe(machine, &op);
+  struct um_op *op = um_op_acquire(machine);
+  um_prep_op(machine, op, OP_ACCEPT, 2, 0);
+  struct io_uring_sqe *sqe = um_get_sqe(machine, op);
   io_uring_prep_accept(sqe, fd, NULL, NULL, 0);
 
   VALUE ret = um_yield(machine);
 
-  if (um_check_completion(machine, &op))
-    ret = INT2NUM(op.result.res);
+  if (likely(um_verify_op_completion(machine, op, false))) ret = INT2NUM(op->result.res);
+  um_op_release(machine, op);
 
   RAISE_IF_EXCEPTION(ret);
   RB_GC_GUARD(ret);
@@ -707,15 +771,15 @@ VALUE um_accept(struct um *machine, int fd) {
 }
 
 VALUE um_socket(struct um *machine, int domain, int type, int protocol, uint flags) {
-  struct um_op op;
-  um_prep_op(machine, &op, OP_SOCKET, 0);
-  struct io_uring_sqe *sqe = um_get_sqe(machine, &op);
+  struct um_op *op = um_op_acquire(machine);
+  um_prep_op(machine, op, OP_SOCKET, 2, 0);
+  struct io_uring_sqe *sqe = um_get_sqe(machine, op);
   io_uring_prep_socket(sqe, domain, type, protocol, flags);
 
   VALUE ret = um_yield(machine);
 
-  if (um_check_completion(machine, &op))
-    ret = INT2NUM(op.result.res);
+  if (likely(um_verify_op_completion(machine, op, false))) ret = INT2NUM(op->result.res);
+  um_op_release(machine, op);
 
   RAISE_IF_EXCEPTION(ret);
   RB_GC_GUARD(ret);
@@ -723,15 +787,15 @@ VALUE um_socket(struct um *machine, int domain, int type, int protocol, uint fla
 }
 
 VALUE um_connect(struct um *machine, int fd, const struct sockaddr *addr, socklen_t addrlen) {
-  struct um_op op;
-  um_prep_op(machine, &op, OP_CONNECT, 0);
-  struct io_uring_sqe *sqe = um_get_sqe(machine, &op);
+  struct um_op *op = um_op_acquire(machine);
+  um_prep_op(machine, op, OP_CONNECT, 2, 0);
+  struct io_uring_sqe *sqe = um_get_sqe(machine, op);
   io_uring_prep_connect(sqe, fd, addr, addrlen);
 
   VALUE ret = um_yield(machine);
 
-  if (um_check_completion(machine, &op))
-    ret = INT2NUM(op.result.res);
+  if (likely(um_verify_op_completion(machine, op, true))) ret = INT2NUM(op->result.res);
+  um_op_release(machine, op);
 
   RAISE_IF_EXCEPTION(ret);
   RB_GC_GUARD(ret);
@@ -739,27 +803,24 @@ VALUE um_connect(struct um *machine, int fd, const struct sockaddr *addr, sockle
 }
 
 VALUE um_send(struct um *machine, int fd, VALUE buffer, size_t len, int flags) {
-  struct um_op op;
-  um_prep_op(machine, &op, OP_SEND, 0);
-  struct io_uring_sqe *sqe = um_get_sqe(machine, &op);
-
   const void *base;
   size_t size;
-  um_get_buffer_bytes_for_writing(buffer, &base, &size);
+  um_get_buffer_bytes_for_writing(buffer, &base, &size, true);
   if ((len == (size_t)-1) || (len > size)) len = size;
 
+  struct um_op *op = um_op_acquire(machine);
+  um_prep_op(machine, op, OP_SEND, 2, 0);
+  struct io_uring_sqe *sqe = um_get_sqe(machine, op);
   io_uring_prep_send(sqe, fd, base, len, flags);
 
   VALUE ret = um_yield(machine);
 
-  if (um_check_completion(machine, &op))
-    ret = INT2NUM(op.result.res);
+  if (likely(um_verify_op_completion(machine, op, true))) ret = INT2NUM(op->result.res);
+  um_op_release(machine, op);
 
   RAISE_IF_EXCEPTION(ret);
   RB_GC_GUARD(ret);
   return ret;
-  // int ret = write(fd, base, len);
-  // return UINT2NUM(ret);
 }
 
 // for some reason we don't get this define from liburing/io_uring.h
@@ -767,22 +828,17 @@ VALUE um_send(struct um *machine, int fd, VALUE buffer, size_t len, int flags) {
 
 VALUE um_sendv(struct um *machine, int fd, int argc, VALUE *argv) {
   struct iovec *iovecs = um_alloc_iovecs_for_writing(argc, argv, NULL);
-  struct um_op op;
-  um_prep_op(machine, &op, OP_SEND, 0);
-  struct io_uring_sqe *sqe = um_get_sqe(machine, &op);
-
+  struct um_op *op = um_op_acquire(machine);
+  op->iovecs = iovecs;
+  um_prep_op(machine, op, OP_SEND, 2, OP_F_FREE_IOVECS);
+  struct io_uring_sqe *sqe = um_get_sqe(machine, op);
   io_uring_prep_send(sqe, fd, iovecs, argc, MSG_NOSIGNAL | MSG_WAITALL);
   sqe->ioprio |= IORING_SEND_VECTORIZED;
 
   VALUE ret = um_yield(machine);
 
-  int completed = um_op_completed_p(&op);
-  if (unlikely(!completed)) um_cancel_and_wait(machine, &op);
-  free(iovecs);
-  if (likely(completed)) {
-    um_raise_on_error_result(op.result.res);
-    ret = INT2NUM(op.result.res);
-  }
+  if (likely(um_verify_op_completion(machine, op, true))) ret = INT2NUM(op->result.res);
+  um_op_release(machine, op);
 
   RAISE_IF_EXCEPTION(ret);
   RB_GC_GUARD(ret);
@@ -791,19 +847,17 @@ VALUE um_sendv(struct um *machine, int fd, int argc, VALUE *argv) {
 
 VALUE um_send_bundle(struct um *machine, int fd, int bgid, VALUE strings) {
   um_add_strings_to_buffer_ring(machine, bgid, strings);
-
-  struct um_op op;
-  um_prep_op(machine, &op, OP_SEND_BUNDLE, 0);
-  struct io_uring_sqe *sqe = um_get_sqe(machine, &op);
-
+  struct um_op *op = um_op_acquire(machine);
+  um_prep_op(machine, op, OP_SEND_BUNDLE, 2, 0);
+  struct io_uring_sqe *sqe = um_get_sqe(machine, op);
 	io_uring_prep_send_bundle(sqe, fd, 0, MSG_NOSIGNAL | MSG_WAITALL);
 	sqe->flags |= IOSQE_BUFFER_SELECT;
 	sqe->buf_group = bgid;
 
   VALUE ret = um_yield(machine);
 
-  if (um_check_completion(machine, &op))
-    ret = INT2NUM(op.result.res);
+  if (likely(um_verify_op_completion(machine, op, true))) ret = INT2NUM(op->result.res);
+  um_op_release(machine, op);
 
   RAISE_IF_EXCEPTION(ret);
   RB_GC_GUARD(ret);
@@ -811,19 +865,19 @@ VALUE um_send_bundle(struct um *machine, int fd, int bgid, VALUE strings) {
 }
 
 VALUE um_recv(struct um *machine, int fd, VALUE buffer, size_t maxlen, int flags) {
-  struct um_op op;
-  um_prep_op(machine, &op, OP_RECV, 0);
-  struct io_uring_sqe *sqe = um_get_sqe(machine, &op);
   void *ptr = um_prepare_read_buffer(buffer, maxlen, 0);
-
+  struct um_op *op = um_op_acquire(machine);
+  um_prep_op(machine, op, OP_RECV, 2, 0);
+  struct io_uring_sqe *sqe = um_get_sqe(machine, op);
   io_uring_prep_recv(sqe, fd, ptr, maxlen, flags);
 
   VALUE ret = um_yield(machine);
 
-  if (um_check_completion(machine, &op)) {
-    um_update_read_buffer(buffer, 0, op.result.res);
-    ret = INT2NUM(op.result.res);
+  if (likely(um_verify_op_completion(machine, op, true))) {
+    um_update_read_buffer(buffer, 0, op->result.res);
+    ret = INT2NUM(op->result.res);
   }
+  um_op_release(machine, op);
 
   RAISE_IF_EXCEPTION(ret);
   RB_GC_GUARD(ret);
@@ -831,15 +885,15 @@ VALUE um_recv(struct um *machine, int fd, VALUE buffer, size_t maxlen, int flags
 }
 
 VALUE um_bind(struct um *machine, int fd, struct sockaddr *addr, socklen_t addrlen) {
-  struct um_op op;
-  um_prep_op(machine, &op, OP_BIND, 0);
-  struct io_uring_sqe *sqe = um_get_sqe(machine, &op);
+  struct um_op *op = um_op_acquire(machine);
+  um_prep_op(machine, op, OP_BIND, 2, 0);
+  struct io_uring_sqe *sqe = um_get_sqe(machine, op);
   io_uring_prep_bind(sqe, fd, addr, addrlen);
 
   VALUE ret = um_yield(machine);
 
-  if (um_check_completion(machine, &op))
-    ret = INT2NUM(op.result.res);
+  if (likely(um_verify_op_completion(machine, op, true))) ret = INT2NUM(op->result.res);
+  um_op_release(machine, op);
 
   RAISE_IF_EXCEPTION(ret);
   RB_GC_GUARD(ret);
@@ -847,15 +901,15 @@ VALUE um_bind(struct um *machine, int fd, struct sockaddr *addr, socklen_t addrl
 }
 
 VALUE um_listen(struct um *machine, int fd, int backlog) {
-  struct um_op op;
-  um_prep_op(machine, &op, OP_BIND, 0);
-  struct io_uring_sqe *sqe = um_get_sqe(machine, &op);
+  struct um_op *op = um_op_acquire(machine);
+  um_prep_op(machine, op, OP_BIND, 2, 0);
+  struct io_uring_sqe *sqe = um_get_sqe(machine, op);
   io_uring_prep_listen(sqe, fd, backlog);
 
   VALUE ret = um_yield(machine);
 
-  if (um_check_completion(machine, &op))
-    ret = INT2NUM(op.result.res);
+  if (likely(um_verify_op_completion(machine, op, false))) ret = INT2NUM(op->result.res);
+  um_op_release(machine, op);
 
   RAISE_IF_EXCEPTION(ret);
   RB_GC_GUARD(ret);
@@ -864,17 +918,15 @@ VALUE um_listen(struct um *machine, int fd, int backlog) {
 
 VALUE um_getsockopt(struct um *machine, int fd, int level, int opt) {
   VALUE ret = Qnil;
-  int value;
-
-  struct um_op op;
-  um_prep_op(machine, &op, OP_GETSOCKOPT, 0);
-  struct io_uring_sqe *sqe = um_get_sqe(machine, &op);
-  io_uring_prep_cmd_sock(sqe, SOCKET_URING_OP_GETSOCKOPT, fd, level, opt, &value, sizeof(value));
+  struct um_op *op = um_op_acquire(machine);
+  um_prep_op(machine, op, OP_GETSOCKOPT, 2, 0);
+  struct io_uring_sqe *sqe = um_get_sqe(machine, op);
+  io_uring_prep_cmd_sock(sqe, SOCKET_URING_OP_GETSOCKOPT, fd, level, opt, &op->int_value, sizeof(op->int_value));
 
   ret = um_yield(machine);
 
-  if (um_check_completion(machine, &op))
-    ret = INT2NUM(value);
+  if (likely(um_verify_op_completion(machine, op, true))) ret = INT2NUM(op->int_value);
+  um_op_release(machine, op);
 
   RAISE_IF_EXCEPTION(ret);
   RB_GC_GUARD(ret);
@@ -883,16 +935,16 @@ VALUE um_getsockopt(struct um *machine, int fd, int level, int opt) {
 
 VALUE um_setsockopt(struct um *machine, int fd, int level, int opt, int value) {
   VALUE ret = Qnil;
-
-  struct um_op op;
-  um_prep_op(machine, &op, OP_SETSOCKOPT, 0);
-  struct io_uring_sqe *sqe = um_get_sqe(machine, &op);
-  io_uring_prep_cmd_sock(sqe, SOCKET_URING_OP_SETSOCKOPT, fd, level, opt, &value, sizeof(value));
+  struct um_op *op = um_op_acquire(machine);
+  um_prep_op(machine, op, OP_SETSOCKOPT, 2, 0);
+  struct io_uring_sqe *sqe = um_get_sqe(machine, op);
+  op->int_value = value;
+  io_uring_prep_cmd_sock(sqe, SOCKET_URING_OP_SETSOCKOPT, fd, level, opt, &op->int_value, sizeof(op->int_value));
 
   ret = um_yield(machine);
 
-  if (um_check_completion(machine, &op))
-    ret = INT2NUM(op.result.res);
+  if (likely(um_verify_op_completion(machine, op, true))) ret = INT2NUM(op->result.res);
+  um_op_release(machine, op);
 
   RAISE_IF_EXCEPTION(ret);
   RB_GC_GUARD(ret);
@@ -901,16 +953,15 @@ VALUE um_setsockopt(struct um *machine, int fd, int level, int opt, int value) {
 
 VALUE um_shutdown(struct um *machine, int fd, int how) {
   VALUE ret = Qnil;
-
-  struct um_op op;
-  um_prep_op(machine, &op, OP_SHUTDOWN, 0);
-  struct io_uring_sqe *sqe = um_get_sqe(machine, &op);
+  struct um_op *op = um_op_acquire(machine);
+  um_prep_op(machine, op, OP_SHUTDOWN, 2, 0);
+  struct io_uring_sqe *sqe = um_get_sqe(machine, op);
   io_uring_prep_shutdown(sqe, fd, how);
 
   ret = um_yield(machine);
 
-  if (um_check_completion(machine, &op))
-    ret = INT2NUM(op.result.res);
+  if (likely(um_verify_op_completion(machine, op, true))) ret = INT2NUM(op->result.res);
+  um_op_release(machine, op);
 
   RAISE_IF_EXCEPTION(ret);
   RB_GC_GUARD(ret);
@@ -918,25 +969,111 @@ VALUE um_shutdown(struct um *machine, int fd, int how) {
 }
 
 VALUE um_shutdown_async(struct um *machine, int fd, int how) {
-  struct um_op *op = um_op_alloc(machine);
-  um_prep_op(machine, op, OP_SHUTDOWN_ASYNC, OP_F_FREE_ON_COMPLETE);
-
+  struct um_op *op = um_op_acquire(machine);
+  um_prep_op(machine, op, OP_SHUTDOWN_ASYNC, 1, OP_F_ASYNC);
   struct io_uring_sqe *sqe = um_get_sqe(machine, op);
   io_uring_prep_shutdown(sqe, fd, how);
 
   return INT2NUM(fd);
 }
 
+struct send_recv_fd_data {
+  struct msghdr msgh;
+  char iobuf[1];
+  struct iovec iov;
+  union { // Ancillary data buffer, wrapped in a union for alignment 
+    char buf[CMSG_SPACE(sizeof(int))];
+    struct cmsghdr align;
+  } u;
+};
+
+inline void recv_fd_prepare(struct send_recv_fd_data *data) {
+  memset(&data->msgh, 0, sizeof(data->msgh));
+  data->iov.iov_base = data->iobuf;
+  data->iov.iov_len = sizeof(data->iobuf);
+  data->msgh.msg_iov = &data->iov;
+  data->msgh.msg_iovlen = 1;
+  data->msgh.msg_control = data->u.buf;
+  data->msgh.msg_controllen = sizeof(data->u.buf);
+}
+
+inline void send_fd_prepare(struct send_recv_fd_data *data, int fd) {
+  memset(&data->msgh, 0, sizeof(data->msgh));
+  data->iov.iov_base = data->iobuf;
+  data->iov.iov_len = sizeof(data->iobuf);
+  data->msgh.msg_iov = &data->iov;
+  data->msgh.msg_iovlen = 1;
+  data->msgh.msg_control = data->u.buf;
+  data->msgh.msg_controllen = sizeof(data->u.buf);
+
+  struct cmsghdr *cmsgp = CMSG_FIRSTHDR(&data->msgh);
+  cmsgp->cmsg_level = SOL_SOCKET;
+  cmsgp->cmsg_type = SCM_RIGHTS;
+  cmsgp->cmsg_len = CMSG_LEN(sizeof(fd));
+  memcpy(CMSG_DATA(cmsgp), &fd, sizeof(fd));
+}
+
+VALUE um_send_fd(struct um *machine, int sock_fd, int fd) {
+  struct send_recv_fd_data data;
+  send_fd_prepare(&data, fd);
+
+  VALUE ret = Qnil;
+  struct um_op *op = um_op_acquire(machine);
+  um_prep_op(machine, op, OP_SENDMSG, 2, 0);
+  struct io_uring_sqe *sqe = um_get_sqe(machine, op);
+  io_uring_prep_sendmsg(sqe, sock_fd, &data.msgh, 0);
+
+  ret = um_yield(machine);
+
+  if (likely(um_verify_op_completion(machine, op, true))) ret = INT2NUM(fd);
+  um_op_release(machine, op);
+
+  RAISE_IF_EXCEPTION(ret);
+  RB_GC_GUARD(ret);
+  return ret;
+}
+
+inline int recv_fd_get_fd(struct send_recv_fd_data *data) {
+  int fd;
+  struct cmsghdr *cmsgp = CMSG_FIRSTHDR(&data->msgh);
+
+  if (cmsgp == NULL || cmsgp->cmsg_len != CMSG_LEN(sizeof(int)) ||
+      cmsgp->cmsg_level != SOL_SOCKET || cmsgp->cmsg_type != SCM_RIGHTS
+  ) um_raise_on_error_result(-EINVAL);
+
+  memcpy(&fd, CMSG_DATA(cmsgp), sizeof(int));
+  return fd;
+}
+
+VALUE um_recv_fd(struct um *machine, int sock_fd) {
+  struct send_recv_fd_data data;
+  recv_fd_prepare(&data);
+
+  struct um_op *op = um_op_acquire(machine);
+  um_prep_op(machine, op, OP_RECVMSG, 2, 0);
+  struct io_uring_sqe *sqe = um_get_sqe(machine, op);
+  io_uring_prep_recvmsg(sqe, sock_fd, &data.msgh, 0);
+
+  VALUE ret = um_yield(machine);
+
+  if (likely(um_verify_op_completion(machine, op, true))) ret = INT2NUM(recv_fd_get_fd(&data));
+  um_op_release(machine, op);
+
+  RAISE_IF_EXCEPTION(ret);
+  RB_GC_GUARD(ret);
+  return ret;
+}
+
 VALUE um_open(struct um *machine, VALUE pathname, int flags, int mode) {
-  struct um_op op;
-  um_prep_op(machine, &op, OP_OPEN, 0);
-  struct io_uring_sqe *sqe = um_get_sqe(machine, &op);
+  struct um_op *op = um_op_acquire(machine);
+  um_prep_op(machine, op, OP_OPEN, 2, 0);
+  struct io_uring_sqe *sqe = um_get_sqe(machine, op);
   io_uring_prep_open(sqe, StringValueCStr(pathname), flags, mode);
 
   VALUE ret = um_yield(machine);
 
-  if (um_check_completion(machine, &op))
-    ret = INT2NUM(op.result.res);
+  if (likely(um_verify_op_completion(machine, op, true))) ret = INT2NUM(op->result.res);
+  um_op_release(machine, op);
 
   RAISE_IF_EXCEPTION(ret);
   RB_GC_GUARD(ret);
@@ -944,27 +1081,25 @@ VALUE um_open(struct um *machine, VALUE pathname, int flags, int mode) {
 }
 
 VALUE um_poll(struct um *machine, int fd, unsigned mask) {
-  struct um_op op;
-  um_prep_op(machine, &op, OP_POLL, 0);
-  struct io_uring_sqe *sqe = um_get_sqe(machine, &op);
+  struct um_op *op = um_op_acquire(machine);
+  um_prep_op(machine, op, OP_POLL, 2, 0);
+  struct io_uring_sqe *sqe = um_get_sqe(machine, op);
   io_uring_prep_poll_add(sqe, fd, mask);
 
   VALUE ret = um_yield(machine);
 
-  if (um_check_completion(machine, &op))
-    ret = INT2NUM(op.result.res);
+  if (likely(um_verify_op_completion(machine, op, false))) ret = INT2NUM(op->result.res);
+  um_op_release(machine, op);
 
   RAISE_IF_EXCEPTION(ret);
   RB_GC_GUARD(ret);
-  RB_GC_GUARD(op.fiber);
-  RB_GC_GUARD(op.value);
   return ret;
 }
 
-static inline void prepare_select_poll_ops(struct um *machine, uint *idx, struct um_op *ops, VALUE fds, uint len, uint flags, uint event) {
+static inline void prepare_select_poll_ops(struct um *machine, uint *idx, struct um_op **ops, VALUE fds, uint len, uint flags, uint event) {
   for (uint i = 0; i < len; i++) {
-    struct um_op *op = ops + ((*idx)++);
-    um_prep_op(machine, op, OP_POLL, flags | OP_F_IGNORE_CANCELED);
+    struct um_op *op = ops[(*idx)++] = um_op_acquire(machine);
+    um_prep_op(machine, op, OP_POLL, 2, flags);
     struct io_uring_sqe *sqe = um_get_sqe(machine, op);
     VALUE fd = rb_ary_entry(fds, i);
     RB_OBJ_WRITE(machine->self, &op->value, fd);
@@ -973,8 +1108,9 @@ static inline void prepare_select_poll_ops(struct um *machine, uint *idx, struct
 }
 
 VALUE um_select_single(struct um *machine, VALUE rfds, VALUE wfds, VALUE efds, uint rfds_len, uint wfds_len, uint efds_len) {
-  struct um_op op;
+  struct um_op *op;
   uint idx = 0;
+
   if (rfds_len)
     prepare_select_poll_ops(machine, &idx, &op, rfds, rfds_len, OP_F_SELECT_POLLIN, POLLIN);
   else if (wfds_len)
@@ -985,12 +1121,14 @@ VALUE um_select_single(struct um *machine, VALUE rfds, VALUE wfds, VALUE efds, u
 
   VALUE ret = um_yield(machine);
 
-  um_check_completion(machine, &op);
+  um_verify_op_completion(machine, op, false);
+  uint flags = op->flags;
+  um_op_release(machine, op);
   RAISE_IF_EXCEPTION(ret);
 
-  if (op.flags & OP_F_SELECT_POLLIN)
+  if (flags & OP_F_SELECT_POLLIN)
     return rb_ary_new3(3, rb_ary_new3(1, ret), rb_ary_new(), rb_ary_new());
-  else if (op.flags & OP_F_SELECT_POLLOUT)
+  else if (flags & OP_F_SELECT_POLLOUT)
     return rb_ary_new3(3, rb_ary_new(), rb_ary_new3(1, ret), rb_ary_new());
   else
     return rb_ary_new3(3, rb_ary_new(), rb_ary_new(), rb_ary_new3(1, ret));
@@ -1009,7 +1147,7 @@ VALUE um_select(struct um *machine, VALUE rfds, VALUE wfds, VALUE efds) {
   if (unlikely(!total_len))
     return rb_ary_new3(3, rb_ary_new(), rb_ary_new(), rb_ary_new());
 
-  struct um_op *ops = malloc(sizeof(struct um_op) * total_len);
+  struct um_op **ops = malloc(sizeof(struct um_op *) * total_len);
   uint idx = 0;
   prepare_select_poll_ops(machine, &idx, ops, rfds, rfds_len, OP_F_SELECT_POLLIN, POLLIN);
   prepare_select_poll_ops(machine, &idx, ops, wfds, wfds_len, OP_F_SELECT_POLLOUT, POLLOUT);
@@ -1017,50 +1155,35 @@ VALUE um_select(struct um *machine, VALUE rfds, VALUE wfds, VALUE efds) {
   assert(idx == total_len);
 
   VALUE ret = um_yield(machine);
-  if (unlikely(um_value_is_exception_p(ret))) {
-    free(ops);
-    um_raise_exception(ret);
-  }
 
   VALUE rfds_out = rb_ary_new();
   VALUE wfds_out = rb_ary_new();
   VALUE efds_out = rb_ary_new();
 
   int error_code = 0;
-  uint pending = total_len;
   for (uint i = 0; i < total_len; i++) {
-    if (um_op_completed_p(&ops[i])) {
-      ops[i].flags |= OP_F_RUNQUEUE_SKIP;
-      pending--;
+    if (OP_CQE_DONE_P(ops[i])) {
+      if (OP_SCHEDULED_P(ops[i])) ops[i]->flags |= OP_F_SKIP;
 
-      if (unlikely((ops[i].result.res < 0) && !error_code)) {
-        error_code = ops[i].result.res;
+      if (unlikely((ops[i]->result.res < 0) && !error_code)) {
+        error_code = ops[i]->result.res;
       }
       else {
-        if (ops[i].flags & OP_F_SELECT_POLLIN)  rb_ary_push(rfds_out, ops[i].value);
-        if (ops[i].flags & OP_F_SELECT_POLLOUT) rb_ary_push(wfds_out, ops[i].value);
-        if (ops[i].flags & OP_F_SELECT_POLLPRI) rb_ary_push(efds_out, ops[i].value);
+        if (ops[i]->flags & OP_F_SELECT_POLLIN)  rb_ary_push(rfds_out, ops[i]->value);
+        if (ops[i]->flags & OP_F_SELECT_POLLOUT) rb_ary_push(wfds_out, ops[i]->value);
+        if (ops[i]->flags & OP_F_SELECT_POLLPRI) rb_ary_push(efds_out, ops[i]->value);
       }
     }
     else {
-      ops[i].flags |= OP_F_CANCELED;
-      um_cancel_op(machine, &ops[i]);
+      um_cancel_op_and_discard_cqe(machine, ops[i]);
     }
   }
 
-  while (pending) {
-    um_wait_for_and_process_ready_cqes(machine, 1);
-
-    for (uint i = 0; i < total_len; i++) {
-      struct um_op *op = ops + i;
-      if (op->flags & OP_F_CANCELED && um_op_completed_p(op)) {
-        pending--;
-      }
-    }
-  }
+  for (uint i = 0; i < total_len; i++) um_op_release(machine, ops[i]);
   free(ops);
 
-  if (error_code)
+  RAISE_IF_EXCEPTION(ret);
+  if (unlikely(error_code))
     um_raise_on_error_result(error_code);
 
   return rb_ary_new3(3, rfds_out, wfds_out, efds_out);
@@ -1070,44 +1193,49 @@ VALUE um_select(struct um *machine, VALUE rfds, VALUE wfds, VALUE efds) {
   RB_GC_GUARD(efds_out);
 }
 
-VALUE um_waitid(struct um *machine, int idtype, int id, int options) {
-  struct um_op op;
-  um_prep_op(machine, &op, OP_WAITID, 0);
-  struct io_uring_sqe *sqe = um_get_sqe(machine, &op);
+static inline VALUE siginfo_to_array(siginfo_t *info) {
+  return rb_ary_new_from_args(
+    3,
+    INT2NUM(info->si_pid),
+    INT2NUM(info->si_status),
+    INT2NUM(info->si_code)
+  );
+}
 
-  siginfo_t infop;
-  io_uring_prep_waitid(sqe, idtype, id, &infop, options, 0);
+VALUE um_waitid(struct um *machine, int idtype, int id, int options) {
+  struct um_op *op = um_op_acquire(machine);
+  um_prep_op(machine, op, OP_WAITID, 2, 0);
+  struct io_uring_sqe *sqe = um_get_sqe(machine, op);
+  io_uring_prep_waitid(sqe, idtype, id, &op->siginfo, options, 0);
 
   VALUE ret = um_yield(machine);
 
-  if (um_check_completion(machine, &op))
-    ret = INT2NUM(op.result.res);
+  if (likely(um_verify_op_completion(machine, op, false))) ret = INT2NUM(op->result.res);
+  um_op_release(machine, op);
 
   RAISE_IF_EXCEPTION(ret);
   RB_GC_GUARD(ret);
 
-  return rb_ary_new_from_args(
-    3, INT2NUM(infop.si_pid), INT2NUM(infop.si_status), INT2NUM(infop.si_code)
-  );
+  return siginfo_to_array(&op->siginfo);
 }
 
 #ifdef HAVE_RB_PROCESS_STATUS_NEW
 VALUE um_waitid_status(struct um *machine, int idtype, int id, int options) {
-  struct um_op op;
-  um_prep_op(machine, &op, OP_WAITID, 0);
-  struct io_uring_sqe *sqe = um_get_sqe(machine, &op);
-
-  siginfo_t infop;
-  io_uring_prep_waitid(sqe, idtype, id, &infop, options | WNOWAIT, 0);
+  struct um_op *op = um_op_acquire(machine);
+  um_prep_op(machine, op, OP_WAITID, 2, 0);
+  struct io_uring_sqe *sqe = um_get_sqe(machine, op);
+  io_uring_prep_waitid(sqe, idtype, id, &op->siginfo, options | WNOWAIT, 0);
 
   VALUE ret = um_yield(machine);
-  if (um_check_completion(machine, &op))
-    ret = INT2NUM(op.result.res);
+
+  if (likely(um_verify_op_completion(machine, op))) ret = INT2NUM(op->result.res);
+  siginfo_t siginfo = op->siginfo;
+  um_op_release(machine, op);
 
   RAISE_IF_EXCEPTION(ret);
   RB_GC_GUARD(ret);
 
-  return rb_process_status_new(infop.si_pid, (infop.si_status & 0xff) << 8, 0);
+  return rb_process_status_new(siginfo.si_pid, (siginfo.si_status & 0xff) << 8, 0);
 }
 #endif
 
@@ -1137,10 +1265,9 @@ VALUE statx_to_hash(struct statx *stat) {
 VALUE um_statx(struct um *machine, int dirfd, VALUE path, int flags, unsigned int mask) {
   static char empty_path[] = "";
 
-  struct um_op op;
-  um_prep_op(machine, &op, OP_STATX, 0);
-  struct io_uring_sqe *sqe = um_get_sqe(machine, &op);
-
+  struct um_op *op = um_op_acquire(machine);
+  um_prep_op(machine, op, OP_STATX, 2, 0);
+  struct io_uring_sqe *sqe = um_get_sqe(machine, op);
   char *path_ptr = NIL_P(path) ? empty_path : StringValueCStr(path);
   struct statx stat;
   memset(&stat, 0, sizeof(stat));
@@ -1148,8 +1275,8 @@ VALUE um_statx(struct um *machine, int dirfd, VALUE path, int flags, unsigned in
 
   VALUE ret = um_yield(machine);
 
-  if (um_check_completion(machine, &op))
-    ret = INT2NUM(op.result.res);
+  if (likely(um_verify_op_completion(machine, op, true))) ret = INT2NUM(op->result.res);
+  um_op_release(machine, op);
 
   RAISE_IF_EXCEPTION(ret);
   RB_GC_GUARD(ret);
@@ -1168,28 +1295,24 @@ VALUE accept_each_start(VALUE arg) {
 
   while (true) {
     VALUE ret = um_yield(ctx->machine);
-    if (!um_op_completed_p(ctx->op)) {
-      RAISE_IF_EXCEPTION(ret);
-      return ret;
-    }
+
+    RAISE_IF_EXCEPTION(ret);
+    if (unlikely(!OP_CQE_SEEN_P(ctx->op))) return ret;
     RB_GC_GUARD(ret);
 
-    int more = false;
     struct um_op_result *result = &ctx->op->result;
     while (result) {
-      more = (result->flags & IORING_CQE_F_MORE);
-      if (result->res < 0) {
-        um_op_multishot_results_clear(ctx->machine, ctx->op);
+      if (unlikely(result->res < 0)) {
         rb_syserr_fail(-result->res, strerror(-result->res));
       }
       rb_yield(INT2NUM(result->res));
       result = result->next;
     }
+
+    if (OP_CQE_DONE_P(ctx->op)) break;
+
     um_op_multishot_results_clear(ctx->machine, ctx->op);
-    if (more)
-      ctx->op->flags &= ~OP_F_COMPLETED;
-    else
-      break;
+    ctx->op->flags &= ~OP_F_CQE_SEEN;
   }
 
   return Qnil;
@@ -1202,28 +1325,24 @@ VALUE accept_into_queue_start(VALUE arg) {
 
   while (true) {
     VALUE ret = um_yield(ctx->machine);
-    if (!um_op_completed_p(ctx->op)) {
-      RAISE_IF_EXCEPTION(ret);
-      return ret;
-    }
+
+    RAISE_IF_EXCEPTION(ret);
+    if (unlikely(!OP_CQE_SEEN_P(ctx->op))) return ret;
     RB_GC_GUARD(ret);
 
-    int more = false;
     struct um_op_result *result = &ctx->op->result;
     while (result) {
-      more = (result->flags & IORING_CQE_F_MORE);
-      if (result->res < 0) {
-        um_op_multishot_results_clear(ctx->machine, ctx->op);
+      if (unlikely(result->res < 0)) {
         rb_syserr_fail(-result->res, strerror(-result->res));
       }
       um_queue_push(ctx->machine, ctx->queue, INT2NUM(result->res));
       result = result->next;
     }
+
+    if (OP_CQE_DONE_P(ctx->op)) break;
+
     um_op_multishot_results_clear(ctx->machine, ctx->op);
-    if (more)
-      ctx->op->flags &= ~OP_F_COMPLETED;
-    else
-      break;
+    ctx->op->flags &= ~OP_F_CQE_SEEN;
   }
 
   return Qnil;
@@ -1231,14 +1350,8 @@ VALUE accept_into_queue_start(VALUE arg) {
 
 VALUE multishot_complete(VALUE arg) {
   struct op_ctx *ctx = (struct op_ctx *)arg;
-  if (ctx->op->multishot_result_count) {
-    int more = ctx->op->multishot_result_tail->flags & IORING_CQE_F_MORE;
-    if (more)
-      ctx->op->flags &= ~OP_F_COMPLETED;
-    um_op_multishot_results_clear(ctx->machine, ctx->op);
-  }
-  if (!um_op_completed_p(ctx->op))
-    um_cancel_and_wait(ctx->machine, ctx->op);
+  um_verify_op_completion(ctx->machine, ctx->op, true);
+  um_op_release(ctx->machine, ctx->op);
 
   if (ctx->read_buf)
     free(ctx->read_buf);
@@ -1247,19 +1360,20 @@ VALUE multishot_complete(VALUE arg) {
 }
 
 VALUE um_accept_each(struct um *machine, int fd) {
-  struct um_op op;
-  um_prep_op(machine, &op, OP_ACCEPT_MULTISHOT, OP_F_MULTISHOT);
+  struct um_op *op = um_op_acquire(machine);
+  um_prep_op(machine, op, OP_ACCEPT_MULTISHOT, 2, OP_F_MULTISHOT);
 
-  struct op_ctx ctx = { .machine = machine, .op = &op, .fd = fd, .read_buf = NULL };
+  struct op_ctx ctx = { .machine = machine, .op = op, .fd = fd, .read_buf = NULL };
   return rb_ensure(accept_each_start, (VALUE)&ctx, multishot_complete, (VALUE)&ctx);
 }
 
 VALUE um_accept_into_queue(struct um *machine, int fd, VALUE queue) {
-  struct um_op op;
-  um_prep_op(machine, &op, OP_ACCEPT_MULTISHOT, OP_F_MULTISHOT);
+  struct um_queue *queue_data = Queue_data(queue);
+  struct um_op *op = um_op_acquire(machine);
+  um_prep_op(machine, op, OP_ACCEPT_MULTISHOT, 2, OP_F_MULTISHOT);
 
   struct op_ctx ctx = {
-    .machine = machine, .op = &op, .fd = fd, .queue = Queue_data(queue), .read_buf = NULL
+    .machine = machine, .op = op, .fd = fd, .queue = queue_data, .read_buf = NULL
   };
   return rb_ensure(accept_into_queue_start, (VALUE)&ctx, multishot_complete, (VALUE)&ctx);
 }
@@ -1271,15 +1385,13 @@ int um_read_each_singleshot_loop(struct op_ctx *ctx) {
   int total = 0;
 
   while (1) {
-    um_prep_op(ctx->machine, ctx->op, OP_READ, 0);
+    um_prep_op(ctx->machine, ctx->op, OP_READ, 2, 0);
     struct io_uring_sqe *sqe = um_get_sqe(ctx->machine, ctx->op);
     io_uring_prep_read(sqe, ctx->fd, ctx->read_buf, ctx->read_maxlen, -1);
 
     VALUE ret = um_yield(ctx->machine);
-    if (um_op_completed_p(ctx->op)) {
-      um_raise_on_error_result(ctx->op->result.res);
-      if (!ctx->op->result.res) return total;
 
+    if (likely(um_verify_op_completion(ctx->machine, ctx->op, true))) {
       VALUE buf = rb_str_new(ctx->read_buf, ctx->op->result.res);
       total += ctx->op->result.res;
       rb_yield(buf);
@@ -1287,10 +1399,11 @@ int um_read_each_singleshot_loop(struct op_ctx *ctx) {
     }
     else {
       RAISE_IF_EXCEPTION(ret);
-      return ret;
+      return 0;
     }
     RB_GC_GUARD(ret);
   }
+  return 0;
 }
 
 // // returns true if more results are expected
@@ -1337,91 +1450,83 @@ VALUE read_recv_each_start(VALUE arg) {
 
   while (true) {
     VALUE ret = um_yield(ctx->machine);
-    if (!um_op_completed_p(ctx->op)) {
-      RAISE_IF_EXCEPTION(ret);
-      return ret;
-    }
+
+    RAISE_IF_EXCEPTION(ret);
+    if (unlikely(!OP_CQE_SEEN_P(ctx->op))) return ret;
     RB_GC_GUARD(ret);
 
-    int more = false;
     struct um_op_result *result = &ctx->op->result;
     while (result) {
       um_raise_on_error_result(result->res);
 
-      more = (result->flags & IORING_CQE_F_MORE);
       if (!read_recv_each_multishot_process_result(ctx, result, &total))
         return Qnil;
 
       // rb_yield(INT2NUM(result->res));
       result = result->next;
     }
+
+    if (OP_CQE_DONE_P(ctx->op)) break;
+
     um_op_multishot_results_clear(ctx->machine, ctx->op);
-    if (more)
-      ctx->op->flags &= ~OP_F_COMPLETED;
-    else
-      break;
+    ctx->op->flags &= ~OP_F_CQE_SEEN;
   }
 
   return Qnil;
 }
 
 VALUE um_read_each(struct um *machine, int fd, int bgid) {
-  struct um_op op;
-  um_prep_op(machine, &op, OP_READ_MULTISHOT, OP_F_MULTISHOT);
+  struct um_op *op = um_op_acquire(machine);
+  um_prep_op(machine, op, OP_READ_MULTISHOT, 2, OP_F_MULTISHOT);
 
-  struct op_ctx ctx = { .machine = machine, .op = &op, .fd = fd, .bgid = bgid, .read_buf = NULL };
+  struct op_ctx ctx = { .machine = machine, .op = op, .fd = fd, .bgid = bgid, .read_buf = NULL };
   return rb_ensure(read_recv_each_start, (VALUE)&ctx, multishot_complete, (VALUE)&ctx);
 }
 
 VALUE um_recv_each(struct um *machine, int fd, int bgid, int flags) {
-  struct um_op op;
-  um_prep_op(machine, &op, OP_RECV_MULTISHOT, OP_F_MULTISHOT);
+  struct um_op *op = um_op_acquire(machine);
+  um_prep_op(machine, op, OP_RECV_MULTISHOT, 2, OP_F_MULTISHOT);
 
-  struct op_ctx ctx = { .machine = machine, .op = &op, .fd = fd, .bgid = bgid, .read_buf = NULL, .flags = flags };
+  struct op_ctx ctx = { .machine = machine, .op = op, .fd = fd, .bgid = bgid, .read_buf = NULL, .flags = flags };
   return rb_ensure(read_recv_each_start, (VALUE)&ctx, multishot_complete, (VALUE)&ctx);
 }
 
 VALUE periodically_start(VALUE arg) {
   struct op_ctx *ctx = (struct op_ctx *)arg;
   struct io_uring_sqe *sqe = um_get_sqe(ctx->machine, ctx->op);
-  io_uring_prep_timeout(sqe, &ctx->ts, 0, IORING_TIMEOUT_MULTISHOT);
+  io_uring_prep_timeout(sqe, &ctx->op->ts, 0, IORING_TIMEOUT_MULTISHOT);
 
   while (true) {
     VALUE ret = um_switch(ctx->machine);
-    if (!um_op_completed_p(ctx->op)) {
-      RAISE_IF_EXCEPTION(ret);
-      return ret;
-    }
+
+    RAISE_IF_EXCEPTION(ret);
+    if (unlikely(!OP_CQE_SEEN_P(ctx->op))) return ret;
     RB_GC_GUARD(ret);
 
-    int more = false;
     struct um_op_result *result = &ctx->op->result;
     while (result) {
-      more = (result->flags & IORING_CQE_F_MORE);
-      if (result->res < 0 && result->res != -ETIME) {
-        um_op_multishot_results_clear(ctx->machine, ctx->op);
-        return Qnil;
-      }
+      if (unlikely(result->res < 0 && result->res != -ETIME)) um_raise_on_error_result(result->res);
+
       rb_yield(Qnil);
       result = result->next;
     }
+    if (OP_CQE_DONE_P(ctx->op)) break;
+
     um_op_multishot_results_clear(ctx->machine, ctx->op);
-    if (more)
-      ctx->op->flags &= ~OP_F_COMPLETED;
-    else
-      break;
+    ctx->op->flags &= ~OP_F_CQE_SEEN;
   }
 
   return Qnil;
 }
 
 VALUE um_periodically(struct um *machine, double interval) {
-  struct um_op op;
-  um_prep_op(machine, &op, OP_SLEEP_MULTISHOT, OP_F_MULTISHOT);
-  op.ts = um_double_to_timespec(interval);
+  struct um_op *op = um_op_acquire(machine);
+  um_prep_op(machine, op, OP_TIMEOUT_MULTISHOT, 2, OP_F_MULTISHOT);
+  op->ts = um_double_to_timespec(interval);
 
-  struct op_ctx ctx = { .machine = machine, .op = &op, .ts = op.ts, .read_buf = NULL };
+  struct op_ctx ctx = { .machine = machine, .op = op, .read_buf = NULL };
   return rb_ensure(periodically_start, (VALUE)&ctx, multishot_complete, (VALUE)&ctx);
+  return Qnil;
 }
 
 extern VALUE SYM_size;