uringmachine 0.20.0 → 0.22.0

data/ext/um/um.h

@@ -5,11 +5,22 @@
 #include <liburing.h>
 
 // debugging
+enum {
+  // set to 1 to enable debug logging
+  DEBUG = 0
+};
+
 #define OBJ_ID(obj) (NUM2LONG(rb_funcall(obj, rb_intern("object_id"), 0)))
 #define INSPECT(str, obj) { printf(str); VALUE s = rb_funcall(obj, rb_intern("inspect"), 0); printf(": %s\n", StringValueCStr(s)); }
 #define CALLER() rb_funcall(rb_mKernel, rb_intern("caller"), 0)
 #define TRACE_CALLER() INSPECT("caller: ", CALLER())
 #define TRACE_FREE(ptr) //printf("Free %p %s:%d\n", ptr, __FILE__, __LINE__)
+#define DEBUG_MARK(machine, markv, msg) \
+  if (machine->mark == markv) printf("%s\n", msg);
+#define DEBUG_PRINTF(...) \
+  if (DEBUG) fprintf(stderr, __VA_ARGS__)
+
+#define SYM_DEF(name) SYM_#name = ID2SYM(rb_intern("#name"))
 
 // branching
 #ifndef unlikely
@@ -23,7 +34,8 @@
 #define IO_BUFFER_P(buffer) \
   (TYPE(buffer) == RUBY_T_DATA) && rb_obj_is_instance_of(buffer, rb_cIOBuffer)
 
-enum op_kind {
+enum um_op_kind {
+  OP_UNDEFINED,
   OP_TIMEOUT,
   OP_SCHEDULE,
 
@@ -62,12 +74,17 @@ enum op_kind {
   OP_SLEEP_MULTISHOT
 };
 
-#define OP_F_COMPLETED        (1U << 0) // op is completed (set on each CQE for multishot ops)
-#define OP_F_TRANSIENT        (1U << 1) // op is heap allocated
-#define OP_F_ASYNC            (1U << 2) // op belongs to an AsyncOp
-#define OP_F_IGNORE_CANCELED  (1U << 3) // CQE with -ECANCEL should be ignored
-#define OP_F_MULTISHOT        (1U << 4) // op is multishot
-#define OP_F_FREE_ON_COMPLETE (1U << 5) // op should be freed on receiving CQE
+#define OP_F_COMPLETED        (1U <<  0) // op is completed (set on each CQE for multishot ops)
+#define OP_F_TRANSIENT        (1U <<  1) // op is heap allocated
+#define OP_F_ASYNC            (1U <<  2) // op belongs to an AsyncOp
+#define OP_F_CANCELED         (1U <<  3) // op is cancelled
+#define OP_F_IGNORE_CANCELED  (1U <<  4) // CQE with -ECANCEL should be ignored
+#define OP_F_MULTISHOT        (1U <<  5) // op is multishot
+#define OP_F_FREE_ON_COMPLETE (1U <<  6) // op should be freed on receiving CQE
+#define OP_F_RUNQUEUE_SKIP    (1U <<  7) // runqueue entry should be skipped
+#define OP_F_SELECT_POLLIN    (1U <<  8) // select POLLIN
+#define OP_F_SELECT_POLLOUT   (1U <<  9) // select POLLOUT
+#define OP_F_SELECT_POLLPRI   (1U << 10) // select POLLPRI
 
 struct um_op_result {
   __s32 res;
@@ -79,8 +96,8 @@ struct um_op {
   struct um_op *prev;
   struct um_op *next;
 
-  enum op_kind kind;
-  unsigned flags;
+  enum um_op_kind kind;
+  uint flags;
 
   VALUE fiber;
   VALUE value;
@@ -88,7 +105,7 @@ struct um_op {
 
   struct um_op_result result;
   struct um_op_result *multishot_result_tail;
-  unsigned multishot_result_count;
+  uint multishot_result_count;
 
   struct __kernel_timespec ts; // used for timeout operation
 };
@@ -108,6 +125,22 @@ struct buf_ring_descriptor {
 	void *buf_base;
 };
 
+struct um_metrics {
+  ulong total_ops;        // total ops submitted
+  ulong total_switches;   // total fiber switches
+  ulong total_waits;      // total number of CQE waits
+
+  uint ops_pending;       // number of pending ops
+  uint ops_unsubmitted;   // number of unsubmitted
+  uint ops_runqueue;      // number of ops in runqueue
+  uint ops_free;          // number of ops in freelist
+  uint ops_transient;     // number of ops in transient list
+
+  double time_total_wait; // total CPU time waiting for CQEs
+  double time_last_cpu; // last seen time stamp
+  double time_first_cpu; // last seen time stamp
+};
+
 #define BUFFER_RING_MAX_COUNT 10
 
 struct um {
@@ -117,14 +150,22 @@ struct um {
 
   struct io_uring ring;
 
-  unsigned int    ring_initialized;
-  unsigned int    unsubmitted_count;
-  unsigned int    pending_count;
+  uint ring_initialized; // is the ring initialized successfully
+  uint mark; // used to mark instances for debugging
+
+  struct um_metrics metrics;
+  int profile_mode;
+
+  uint buffer_ring_count; // number of registered buffer rings
+
+  uint size; // size of SQ
+  uint sqpoll_mode; // SQPOLL mode enabled
 
   struct buf_ring_descriptor buffer_rings[BUFFER_RING_MAX_COUNT];
-  unsigned int buffer_ring_count;
 
-  struct um_op *transient_head;
+  struct um_op *transient_head; // list of pending transient ops
+  VALUE pending_fibers; // hash containing pending fibers
+
   struct um_op *runqueue_head;
   struct um_op *runqueue_tail;
 
@@ -184,9 +225,12 @@ extern VALUE cAsyncOp;
 extern VALUE eStreamRESPError;
 
 struct um *um_get_machine(VALUE self);
-void um_setup(VALUE self, struct um *machine);
+void um_setup(VALUE self, struct um *machine, uint size, uint sqpoll_timeout_msec);
 void um_teardown(struct um *machine);
 
+VALUE um_metrics(struct um *machine, struct um_metrics *metrics);
+
+const char * um_op_kind_name(enum um_op_kind kind);
 struct um_op *um_op_alloc(struct um *machine);
 void um_op_free(struct um *machine, struct um_op *op);
 void um_op_clear(struct um *machine, struct um_op *op);
@@ -207,12 +251,14 @@ void um_free_buffer_linked_list(struct um *machine);
 
 struct __kernel_timespec um_double_to_timespec(double value);
 double um_timestamp_to_double(__s64 tv_sec, __u32 tv_nsec);
+double um_get_time_cpu();
+double um_get_time_monotonic();
 int um_value_is_exception_p(VALUE v);
 VALUE um_raise_exception(VALUE v);
 
-#define RAISE_IF_EXCEPTION(v) if (um_value_is_exception_p(v)) { um_raise_exception(v); }
+#define RAISE_IF_EXCEPTION(v) if (unlikely(um_value_is_exception_p(v))) { um_raise_exception(v); }
 
-void um_prep_op(struct um *machine, struct um_op *op, enum op_kind kind, unsigned flags);
+void um_prep_op(struct um *machine, struct um_op *op, enum um_op_kind kind, unsigned flags);
 void um_raise_on_error_result(int result);
 void um_get_buffer_bytes_for_writing(VALUE buffer, const void **base, size_t *size);
 void * um_prepare_read_buffer(VALUE buffer, ssize_t len, ssize_t ofs);
@@ -223,9 +269,11 @@ void um_add_strings_to_buffer_ring(struct um *machine, int bgid, VALUE strings);
 
 struct io_uring_sqe *um_get_sqe(struct um *machine, struct um_op *op);
 
-VALUE um_fiber_switch(struct um *machine);
-VALUE um_await(struct um *machine);
-void um_submit_cancel_op(struct um *machine, struct um_op *op);
+uint um_submit(struct um *machine);
+VALUE um_yield(struct um *machine);
+VALUE um_switch(struct um *machine);
+VALUE um_wakeup(struct um *machine);
+void um_cancel_op(struct um *machine, struct um_op *op);
 void um_cancel_and_wait(struct um *machine, struct um_op *op);
 int um_check_completion(struct um *machine, struct um_op *op);
 
@@ -236,17 +284,22 @@ VALUE um_timeout(struct um *machine, VALUE interval, VALUE class);
 
 VALUE um_sleep(struct um *machine, double duration);
 VALUE um_periodically(struct um *machine, double interval);
-VALUE um_read(struct um *machine, int fd, VALUE buffer, size_t maxlen, ssize_t buffer_offset);
+VALUE um_read(struct um *machine, int fd, VALUE buffer, size_t maxlen, ssize_t buffer_offset, __u64 file_offset);
 size_t um_read_raw(struct um *machine, int fd, char *buffer, size_t maxlen);
 VALUE um_read_each(struct um *machine, int fd, int bgid);
-VALUE um_write(struct um *machine, int fd, VALUE buffer, size_t len);
-VALUE um_write_async(struct um *machine, int fd, VALUE buffer);
+VALUE um_write(struct um *machine, int fd, VALUE buffer, size_t len, __u64 file_offset);
+VALUE um_write_async(struct um *machine, int fd, VALUE buffer, size_t len, __u64 file_offset);
 VALUE um_close(struct um *machine, int fd);
 VALUE um_close_async(struct um *machine, int fd);
 VALUE um_open(struct um *machine, VALUE pathname, int flags, int mode);
 VALUE um_poll(struct um *machine, int fd, unsigned mask);
+VALUE um_select(struct um *machine, VALUE rfds, VALUE wfds, VALUE efds);
 VALUE um_waitid(struct um *machine, int idtype, int id, int options);
+
+#ifdef HAVE_RB_PROCESS_STATUS_NEW
 VALUE um_waitid_status(struct um *machine, int idtype, int id, int options);
+#endif
+
 VALUE um_statx(struct um *machine, int dirfd, VALUE path, int flags, unsigned int mask);
 
 VALUE um_accept(struct um *machine, int fd);

data/ext/um/um_async_op.c

@@ -26,7 +26,7 @@ VALUE um_async_op_await(struct um_async_op *async_op) {
   RB_OBJ_WRITE(async_op->machine->self, &async_op->op->fiber, rb_fiber_current());
   async_op->op->flags &= ~OP_F_ASYNC;
 
-  VALUE ret = um_fiber_switch(async_op->machine);
+  VALUE ret = um_switch(async_op->machine);
   if (!um_op_completed_p(async_op->op))
     um_cancel_and_wait(async_op->machine, async_op->op);
 
@@ -36,5 +36,5 @@ VALUE um_async_op_await(struct um_async_op *async_op) {
 }
 
 void um_async_op_cancel(struct um_async_op *async_op) {
-  um_submit_cancel_op(async_op->machine, async_op->op);
+  um_cancel_op(async_op->machine, async_op->op);
 }

data/ext/um/um_class.c

@@ -3,15 +3,29 @@
 #include <ruby/io.h>
 #include <sys/syscall.h>
 #include <unistd.h>
+#include <sys/socket.h>
 
 VALUE cUM;
 VALUE eUMError;
 
+VALUE SYM_size;
+VALUE SYM_total_ops;
+VALUE SYM_total_switches;
+VALUE SYM_total_waits;
+VALUE SYM_ops_pending;
+VALUE SYM_ops_unsubmitted;
+VALUE SYM_ops_runqueue;
+VALUE SYM_ops_free;
+VALUE SYM_ops_transient;
+VALUE SYM_time_total_cpu;
+VALUE SYM_time_total_wait;
+
 static ID id_fileno;
 
 static void UM_mark(void *ptr) {
   struct um *machine = ptr;
   rb_gc_mark_movable(machine->self);
+  rb_gc_mark_movable(machine->pending_fibers);
 
   um_op_list_mark(machine, machine->transient_head);
   um_op_list_mark(machine, machine->runqueue_head);
@@ -20,6 +34,7 @@ static void UM_mark(void *ptr) {
 static void UM_compact(void *ptr) {
   struct um *machine = ptr;
   machine->self = rb_gc_location(machine->self);
+  machine->pending_fibers = rb_gc_location(machine->pending_fibers);
 
   um_op_list_compact(machine, machine->transient_head);
   um_op_list_compact(machine, machine->runqueue_head);
@@ -54,9 +69,33 @@ inline struct um *um_get_machine(VALUE self) {
   return um;
 }
 
-VALUE UM_initialize(VALUE self) {
+static inline uint get_sqpoll_timeout_msec(VALUE sqpoll_timeout) {
+  switch (TYPE(sqpoll_timeout)) {
+    case T_NIL:
+    case T_FALSE:
+      return 0;
+    case T_FLOAT:
+      return (uint)(NUM2DBL(sqpoll_timeout) * 1000);
+    case T_FIXNUM:
+      return NUM2UINT(sqpoll_timeout) * 1000;
+    case T_TRUE:
+      return 1000;
+    default:
+      rb_raise(eUMError, "Invalid sqpoll_timeout value");
+  }
+}
+
+VALUE UM_initialize(int argc, VALUE *argv, VALUE self) {
   struct um *machine = RTYPEDDATA_DATA(self);
-  um_setup(self, machine);
+  VALUE entries;
+  VALUE sqpoll_timeout;
+  rb_scan_args(argc, argv, "02", &entries, &sqpoll_timeout);
+
+  uint entries_i = NIL_P(entries) ? 0 : NUM2UINT(entries);
+  uint sqpoll_timeout_msec = get_sqpoll_timeout_msec(sqpoll_timeout);
+
+
+  um_setup(self, machine, entries_i, sqpoll_timeout_msec);
   return self;
 }
 
@@ -66,20 +105,78 @@ VALUE UM_setup_buffer_ring(VALUE self, VALUE size, VALUE count) {
   return INT2NUM(bgid);
 }
 
-VALUE UM_pending_count(VALUE self) {
+VALUE UM_size(VALUE self) {
+  struct um *machine = um_get_machine(self);
+  return UINT2NUM(machine->size);
+}
+
+VALUE UM_mark_m(VALUE self, VALUE mark) {
   struct um *machine = um_get_machine(self);
-  return INT2NUM(machine->pending_count);
+  machine->mark = NUM2UINT(mark);
+  return self;
+}
+
+VALUE UM_metrics(VALUE self) {
+  struct um *machine = um_get_machine(self);
+  return um_metrics(machine, &machine->metrics);
+}
+
+VALUE UM_profile_p(VALUE self) {
+  struct um *machine = um_get_machine(self);
+  return machine->profile_mode ? Qtrue : Qfalse;
+}
+
+VALUE UM_profile_set(VALUE self, VALUE value) {
+  struct um *machine = um_get_machine(self);
+  machine->profile_mode = RTEST(value);
+  if (machine->profile_mode) {
+    machine->metrics.time_total_wait = 0.0;
+    machine->metrics.time_last_cpu = machine->metrics.time_first_cpu = um_get_time_cpu();
+  }
+  return value;
 }
 
 VALUE UM_snooze(VALUE self) {
   struct um *machine = um_get_machine(self);
   um_schedule(machine, rb_fiber_current(), Qnil);
-  return um_await(machine);
+
+  // the current fiber is already scheduled, and the runqueue is GC-marked, so
+  // we can safely call um_switch, which is faster than calling um_yield.
+  VALUE ret = um_switch(machine);
+  RAISE_IF_EXCEPTION(ret);
+  return ret;
 }
 
 VALUE UM_yield(VALUE self) {
   struct um *machine = um_get_machine(self);
-  return um_await(machine);
+
+  VALUE ret = um_yield(machine);
+  RAISE_IF_EXCEPTION(ret);
+  return ret;
+}
+
+VALUE UM_switch(VALUE self) {
+  struct um *machine = um_get_machine(self);
+
+  VALUE ret = um_switch(machine);
+  RAISE_IF_EXCEPTION(ret);
+  return ret;
+}
+
+VALUE UM_wakeup(VALUE self) {
+  struct um *machine = um_get_machine(self);
+  return um_wakeup(machine);
+}
+
+VALUE UM_submit(VALUE self) {
+  struct um *machine = um_get_machine(self);
+  uint ret = um_submit(machine);
+  return UINT2NUM(ret);
+}
+
+VALUE UM_pending_fibers(VALUE self) {
+  struct um *machine = um_get_machine(self);
+  return machine->pending_fibers;
 }
 
 VALUE UM_schedule(VALUE self, VALUE fiber, VALUE value) {
@@ -109,12 +206,14 @@ VALUE UM_read(int argc, VALUE *argv, VALUE self) {
   VALUE buffer;
   VALUE maxlen;
   VALUE buffer_offset;
-  rb_scan_args(argc, argv, "22", &fd, &buffer, &maxlen, &buffer_offset);
+  VALUE file_offset;
+  rb_scan_args(argc, argv, "23", &fd, &buffer, &maxlen, &buffer_offset, &file_offset);
 
   ssize_t maxlen_i = NIL_P(maxlen) ? -1 : NUM2INT(maxlen);
   ssize_t buffer_offset_i = NIL_P(buffer_offset) ? 0 : NUM2INT(buffer_offset);
+  __u64 file_offset_i = NIL_P(file_offset) ? (__u64)-1 : NUM2UINT(file_offset);
 
-  return um_read(machine, NUM2INT(fd), buffer, maxlen_i, buffer_offset_i);
+  return um_read(machine, NUM2INT(fd), buffer, maxlen_i, buffer_offset_i, file_offset_i);
 }
 
 VALUE UM_read_each(VALUE self, VALUE fd, VALUE bgid) {
@@ -127,15 +226,27 @@ VALUE UM_write(int argc, VALUE *argv, VALUE self) {
   VALUE fd;
   VALUE buffer;
   VALUE len;
-  rb_scan_args(argc, argv, "21", &fd, &buffer, &len);
+  VALUE file_offset;
+  rb_scan_args(argc, argv, "22", &fd, &buffer, &len, &file_offset);
+
+  size_t len_i = NIL_P(len) ? (size_t)-1 : NUM2UINT(len);
+  __u64 file_offset_i = NIL_P(file_offset) ? (__u64)-1 : NUM2UINT(file_offset);
 
-  size_t bytes = NIL_P(len) ? (size_t)-1 : NUM2UINT(len);
-  return um_write(machine, NUM2INT(fd), buffer, bytes);
+  return um_write(machine, NUM2INT(fd), buffer, len_i, file_offset_i);
 }
 
-VALUE UM_write_async(VALUE self, VALUE fd, VALUE buffer) {
+VALUE UM_write_async(int argc, VALUE *argv, VALUE self) {
   struct um *machine = um_get_machine(self);
-  return um_write_async(machine, NUM2INT(fd), buffer);
+  VALUE fd;
+  VALUE buffer;
+  VALUE len;
+  VALUE file_offset;
+  rb_scan_args(argc, argv, "22", &fd, &buffer, &len, &file_offset);
+
+  size_t len_i = NIL_P(len) ? (size_t)-1 : NUM2UINT(len);
+  __u64 file_offset_i = NIL_P(file_offset) ? (__u64)-1 : NUM2UINT(file_offset);
+
+  return um_write_async(machine, NUM2INT(fd), buffer, len_i, file_offset_i);
 }
 
 VALUE UM_statx(VALUE self, VALUE dirfd, VALUE path, VALUE flags, VALUE mask) {
@@ -330,6 +441,11 @@ VALUE UM_poll(VALUE self, VALUE fd, VALUE mask) {
   return um_poll(machine, NUM2INT(fd), NUM2UINT(mask));
 }
 
+VALUE UM_select(VALUE self, VALUE rfds, VALUE wfds, VALUE efds) {
+  struct um *machine = um_get_machine(self);
+  return um_select(machine, rfds, wfds, efds);
+}
+
 VALUE UM_waitid(VALUE self, VALUE idtype, VALUE id, VALUE options) {
   struct um *machine = um_get_machine(self);
   return um_waitid(machine, NUM2INT(idtype), NUM2INT(id), NUM2INT(options));
@@ -358,6 +474,17 @@ VALUE UM_pipe(VALUE self) {
   return rb_ary_new_from_args(2, INT2NUM(fds[0]), INT2NUM(fds[1]));
 }
 
+VALUE UM_socketpair(VALUE self, VALUE domain, VALUE type, VALUE protocol) {
+  int fds[2];
+  int ret = socketpair(NUM2INT(domain), NUM2INT(type), NUM2INT(protocol), fds);
+  if (ret) {
+    int e = errno;
+    rb_syserr_fail(e, strerror(e));
+  }
+
+  return rb_ary_new_from_args(2, INT2NUM(fds[0]), INT2NUM(fds[1]));
+}
+
 VALUE UM_pidfd_open(VALUE self, VALUE pid) {
   int fd = syscall(SYS_pidfd_open, NUM2INT(pid), 0);
   if (fd == -1) {
@@ -413,7 +540,7 @@ VALUE UM_kernel_version(VALUE self) {
 }
 
 VALUE UM_debug(VALUE self, VALUE str) {
-  printf("%s\n", StringValueCStr(str));
+  fprintf(stderr, "%s\n", StringValueCStr(str));
   return Qnil;
 }
 
@@ -423,11 +550,17 @@ void Init_UM(void) {
   cUM = rb_define_class("UringMachine", rb_cObject);
   rb_define_alloc_func(cUM, UM_allocate);
 
-  rb_define_method(cUM, "initialize", UM_initialize, 0);
-  rb_define_method(cUM, "pending_count", UM_pending_count, 0);
+  rb_define_method(cUM, "initialize", UM_initialize, -1);
+  rb_define_method(cUM, "size", UM_size, 0);
+  rb_define_method(cUM, "mark", UM_mark_m, 1);
+  rb_define_method(cUM, "metrics", UM_metrics, 0);
+  rb_define_method(cUM, "profile?", UM_profile_p, 0);
+  rb_define_method(cUM, "profile", UM_profile_set, 1);
+
   rb_define_method(cUM, "setup_buffer_ring", UM_setup_buffer_ring, 2);
 
   rb_define_singleton_method(cUM, "pipe", UM_pipe, 0);
+  rb_define_singleton_method(cUM, "socketpair", UM_socketpair, 3);
   rb_define_singleton_method(cUM, "pidfd_open", UM_pidfd_open, 1);
   rb_define_singleton_method(cUM, "pidfd_send_signal", UM_pidfd_send_signal, 2);
 
@@ -436,11 +569,14 @@ void Init_UM(void) {
   rb_define_singleton_method(cUM, "kernel_version", UM_kernel_version, 0);
   rb_define_singleton_method(cUM, "debug", UM_debug, 1);
 
-
   rb_define_method(cUM, "schedule", UM_schedule, 2);
   rb_define_method(cUM, "snooze", UM_snooze, 0);
   rb_define_method(cUM, "timeout", UM_timeout, 2);
   rb_define_method(cUM, "yield", UM_yield, 0);
+  rb_define_method(cUM, "switch", UM_switch, 0);
+  rb_define_method(cUM, "wakeup", UM_wakeup, 0);
+  rb_define_method(cUM, "submit", UM_submit, 0);
+  rb_define_method(cUM, "pending_fibers", UM_pending_fibers, 0);
 
   rb_define_method(cUM, "close", UM_close, 1);
   rb_define_method(cUM, "close_async", UM_close_async, 1);
@@ -450,11 +586,13 @@ void Init_UM(void) {
   rb_define_method(cUM, "sleep", UM_sleep, 1);
   rb_define_method(cUM, "periodically", UM_periodically, 1);
   rb_define_method(cUM, "write", UM_write, -1);
-  rb_define_method(cUM, "write_async", UM_write_async, 2);
+  rb_define_method(cUM, "write_async", UM_write_async, -1);
   rb_define_method(cUM, "statx", UM_statx, 4);
 
   rb_define_method(cUM, "poll", UM_poll, 2);
+  rb_define_method(cUM, "select", UM_select, 3);
   rb_define_method(cUM, "waitid", UM_waitid, 3);
+
   #ifdef HAVE_RB_PROCESS_STATUS_NEW
   rb_define_method(cUM, "waitid_status", UM_waitid_status, 3);
   #endif
@@ -486,5 +624,17 @@ void Init_UM(void) {
 
   um_define_net_constants(cUM);
 
+  SYM_size =            ID2SYM(rb_intern("size"));
+  SYM_total_ops =       ID2SYM(rb_intern("total_ops"));
+  SYM_total_switches =  ID2SYM(rb_intern("total_switches"));
+  SYM_total_waits =     ID2SYM(rb_intern("total_waits"));
+  SYM_ops_pending =     ID2SYM(rb_intern("ops_pending"));
+  SYM_ops_unsubmitted = ID2SYM(rb_intern("ops_unsubmitted"));
+  SYM_ops_runqueue =    ID2SYM(rb_intern("ops_runqueue"));
+  SYM_ops_free =        ID2SYM(rb_intern("ops_free"));
+  SYM_ops_transient =   ID2SYM(rb_intern("ops_transient"));
+  SYM_time_total_cpu =  ID2SYM(rb_intern("time_total_cpu"));
+  SYM_time_total_wait = ID2SYM(rb_intern("time_total_wait"));
+
   id_fileno = rb_intern_const("fileno");
 }

data/ext/um/um_op.c

@@ -1,5 +1,42 @@
 #include "um.h"
 
+const char * um_op_kind_name(enum um_op_kind kind) {
+  switch (kind) {
+    case OP_TIMEOUT:            return "OP_TIMEOUT";
+    case OP_SCHEDULE:           return "OP_SCHEDULE";
+    case OP_SLEEP:              return "OP_SLEEP";
+    case OP_OPEN:               return "OP_OPEN";
+    case OP_READ:               return "OP_READ";
+    case OP_WRITE:              return "OP_WRITE";
+    case OP_WRITE_ASYNC:        return "OP_WRITE_ASYNC";
+    case OP_CLOSE:              return "OP_CLOSE";
+    case OP_CLOSE_ASYNC:        return "OP_CLOSE_ASYNC";
+    case OP_STATX:              return "OP_STATX";
+    case OP_ACCEPT:             return "OP_ACCEPT";
+    case OP_RECV:               return "OP_RECV";
+    case OP_SEND:               return "OP_SEND";
+    case OP_SEND_BUNDLE:        return "OP_SEND_BUNDLE";
+    case OP_SOCKET:             return "OP_SOCKET";
+    case OP_CONNECT:            return "OP_CONNECT";
+    case OP_BIND:               return "OP_BIND";
+    case OP_LISTEN:             return "OP_LISTEN";
+    case OP_GETSOCKOPT:         return "OP_GETSOCKOPT";
+    case OP_SETSOCKOPT:         return "OP_SETSOCKOPT";
+    case OP_SHUTDOWN:           return "OP_SHUTDOWN";
+    case OP_SHUTDOWN_ASYNC:     return "OP_SHUTDOWN_ASYNC";
+    case OP_POLL:               return "OP_POLL";
+    case OP_WAITID:             return "OP_WAITID";
+    case OP_FUTEX_WAIT:         return "OP_FUTEX_WAIT";
+    case OP_FUTEX_WAKE:         return "OP_FUTEX_WAKE";
+    case OP_ACCEPT_MULTISHOT:   return "OP_ACCEPT_MULTISHOT";
+    case OP_READ_MULTISHOT:     return "OP_READ_MULTISHOT";
+    case OP_RECV_MULTISHOT:     return "OP_RECV_MULTISHOT";
+    case OP_TIMEOUT_MULTISHOT:  return "OP_TIMEOUT_MULTISHOT";
+    case OP_SLEEP_MULTISHOT:    return "OP_SLEEP_MULTISHOT";
+    default:                    return "UNKNOWN_OP_KIND";
+  }
+}
+
 inline void um_op_clear(struct um *machine, struct um_op *op) {
   memset(op, 0, sizeof(struct um_op));
   op->fiber = Qnil;
@@ -13,6 +50,7 @@ inline void um_op_transient_add(struct um *machine, struct um_op *op) {
     machine->transient_head->prev = op;
   }
   machine->transient_head = op;
+  machine->metrics.ops_transient++;
 }
 
 inline void um_op_transient_remove(struct um *machine, struct um_op *op) {
@@ -23,6 +61,7 @@ inline void um_op_transient_remove(struct um *machine, struct um_op *op) {
 
   if (machine->transient_head == op)
     machine->transient_head = op->next;
+  machine->metrics.ops_transient--;
 }
 
 inline void um_runqueue_push(struct um *machine, struct um_op *op) {
@@ -34,6 +73,7 @@ inline void um_runqueue_push(struct um *machine, struct um_op *op) {
   else
     machine->runqueue_head = machine->runqueue_tail = op;
   op->next = NULL;
+  machine->metrics.ops_runqueue++;
 }
 
 inline struct um_op *um_runqueue_shift(struct um *machine) {
@@ -43,6 +83,7 @@ inline struct um_op *um_runqueue_shift(struct um *machine) {
   machine->runqueue_head = op->next;
   if (!machine->runqueue_head)
     machine->runqueue_tail = NULL;
+  machine->metrics.ops_runqueue--;
   return op;
 }
 
@@ -115,6 +156,7 @@ inline struct um_op *um_op_alloc(struct um *machine) {
   if (machine->op_freelist) {
     struct um_op *op = machine->op_freelist;
     machine->op_freelist = op->next;
+    machine->metrics.ops_free--;
     return op;
   }
   return malloc(sizeof(struct um_op));
@@ -123,4 +165,5 @@ inline struct um_op *um_op_alloc(struct um *machine) {
 inline void um_op_free(struct um *machine, struct um_op *op) {
   op->next = machine->op_freelist;
   machine->op_freelist = op;
+  machine->metrics.ops_free++;
 }

data/ext/um/um_sync.c

@@ -4,16 +4,16 @@
 
 #define FUTEX2_SIZE_U32		0x02
 
-void um_futex_wait(struct um *machine, uint32_t *futex, uint32_t expect) {
+// The value argument is the current (known) futex value.
+void um_futex_wait(struct um *machine, uint32_t *futex, uint32_t value) {
   struct um_op op;
   um_prep_op(machine, &op, OP_FUTEX_WAIT, 0);
   struct io_uring_sqe *sqe = um_get_sqe(machine, &op);
   io_uring_prep_futex_wait(
-    sqe, (uint32_t *)futex, expect, FUTEX_BITSET_MATCH_ANY,
-		FUTEX2_SIZE_U32, 0
+    sqe, (uint32_t *)futex, value, FUTEX_BITSET_MATCH_ANY, FUTEX2_SIZE_U32, 0
   );
 
-  VALUE ret = um_fiber_switch(machine);
+  VALUE ret = um_yield(machine);
   if (!um_op_completed_p(&op))
     um_cancel_and_wait(machine, &op);
   else {
@@ -29,13 +29,11 @@ void um_futex_wake(struct um *machine, uint32_t *futex, uint32_t num_waiters) {
   struct um_op op;
   um_prep_op(machine, &op, OP_FUTEX_WAKE, 0);
   struct io_uring_sqe *sqe = um_get_sqe(machine, &op);
-  // submit futex_wait
   io_uring_prep_futex_wake(
-    sqe, (uint32_t *)futex, num_waiters, FUTEX_BITSET_MATCH_ANY,
-		FUTEX2_SIZE_U32, 0
+    sqe, (uint32_t *)futex, num_waiters, FUTEX_BITSET_MATCH_ANY, FUTEX2_SIZE_U32, 0
   );
 
-  VALUE ret = um_fiber_switch(machine);
+  VALUE ret = um_yield(machine);
   um_check_completion(machine, &op);
 
   RAISE_IF_EXCEPTION(ret);
@@ -45,12 +43,11 @@ void um_futex_wake(struct um *machine, uint32_t *futex, uint32_t num_waiters) {
 void um_futex_wake_transient(struct um *machine, uint32_t *futex, uint32_t num_waiters) {
   struct io_uring_sqe *sqe = um_get_sqe(machine, NULL);
   io_uring_prep_futex_wake(
-    sqe, (uint32_t *)futex, num_waiters, FUTEX_BITSET_MATCH_ANY,
-		FUTEX2_SIZE_U32, 0
+    sqe, (uint32_t *)futex, num_waiters, FUTEX_BITSET_MATCH_ANY, FUTEX2_SIZE_U32, 0
   );
+  um_submit(machine);
 }
 
-
 #define MUTEX_LOCKED    1
 #define MUTEX_UNLOCKED  0
 
@@ -210,7 +207,6 @@ static inline VALUE um_queue_add(struct um *machine, struct um_queue *queue, VAL
   else          queue_add_tail(queue, value);
 
   queue->count++;
-
   queue->state = QUEUE_READY;
   if (queue->num_waiters)
     um_futex_wake_transient(machine, &queue->state, 1);
@@ -241,10 +237,8 @@ VALUE um_queue_remove_start(VALUE arg) {
     um_futex_wait(ctx->machine, &ctx->queue->state, QUEUE_EMPTY);
   }
 
-  if (ctx->queue->state != QUEUE_READY)
-    um_raise_internal_error("Internal error: queue should be in ready state!");
-  if (!ctx->queue->tail)
-    um_raise_internal_error("Internal error: queue should be in ready state!");
+  assert(ctx->queue->state == QUEUE_READY);
+  assert(ctx->queue->tail);
 
   ctx->queue->count--;
   return (ctx->op == QUEUE_POP ? queue_remove_tail : queue_remove_head)(ctx->queue);