uringmachine 0.23.1 → 0.24.0

data/examples/fiber_scheduler_file_io.rb

@@ -0,0 +1,34 @@
+# frozen_string_literal: true
+
+require 'bundler/setup'
+
+require 'uringmachine'
+require 'uringmachine/fiber_scheduler'
+require 'securerandom'
+
+$machine = UringMachine.new
+scheduler = UM::FiberScheduler.new($machine)
+Fiber.set_scheduler scheduler
+
+fn = "/tmp/file_io_#{SecureRandom.hex}"
+
+r, w = IO.pipe
+
+f1 = Fiber.schedule do
+  File.open(fn, 'w') {
+    it.sync = true
+    UM.debug "writing..."
+    it << 'foobar'
+    # w.close
+  }
+
+  File.open(fn, 'r') {
+    UM.debug "reading..."
+    buf = it.read
+    UM.debug "read: #{buf}"
+  }
+rescue => e
+  p e
+  p e.backtrace
+end
+scheduler.join

data/examples/fiber_scheduler_file_io_async.rb

@@ -0,0 +1,33 @@
+# frozen_string_literal: true
+
+require 'bundler/inline'
+
+gemfile do
+  source 'https://rubygems.org'
+  gem 'io-event'
+  gem 'async'
+end
+
+
+require 'async'
+require 'securerandom'
+
+selector ||= IO::Event::Selector::URing.new(Fiber.current)
+scheduler = Async::Scheduler.new(selector:)
+Fiber.set_scheduler scheduler
+
+fn = "/tmp/file_io_#{SecureRandom.hex}"
+
+scheduler.run do
+  Fiber.schedule do
+    File.open(fn, 'w') {
+      it << 'foo'
+      p pre_flush: IO.read(fn)
+      it.flush
+      it << 'bar'
+      p post_flush: IO.read(fn)
+
+    }
+    p post_close: IO.read(fn)
+  end
+end

data/ext/um/um.c

@@ -18,7 +18,7 @@ inline void prepare_io_uring_params(struct io_uring_params *params, uint sqpoll_
     params->flags |= IORING_SETUP_COOP_TASKRUN;
 }
 
-void um_setup(VALUE self, struct um *machine, uint size, uint sqpoll_timeout_msec) {
+void um_setup(VALUE self, struct um *machine, uint size, uint sqpoll_timeout_msec, int sidecar_mode) {
   memset(machine, 0, sizeof(struct um));
 
   RB_OBJ_WRITE(self, &machine->self, self);
@@ -27,16 +27,26 @@ void um_setup(VALUE self, struct um *machine, uint size, uint sqpoll_timeout_mse
   machine->size = (size > 0) ? size : DEFAULT_SIZE;
   machine->sqpoll_mode = !!sqpoll_timeout_msec;
 
+  // sidecar handling
+  machine->sidecar_mode = sidecar_mode;
+  machine->sidecar_signal = aligned_alloc(4, sizeof(uint32_t));
+  memset(machine->sidecar_signal, 0, sizeof(uint32_t));
+
   struct io_uring_params params;
   prepare_io_uring_params(&params, sqpoll_timeout_msec);
   int ret = io_uring_queue_init_params(machine->size, &machine->ring, &params);
   if (ret) rb_syserr_fail(-ret, strerror(-ret));
   machine->ring_initialized = 1;
+
+  if (machine->sidecar_mode) um_sidecar_setup(machine);
 }
 
 inline void um_teardown(struct um *machine) {
   if (!machine->ring_initialized) return;
 
+  if (machine->sidecar_mode) um_sidecar_teardown(machine);
+  if (machine->sidecar_signal) free(machine->sidecar_signal);
+
   for (unsigned i = 0; i < machine->buffer_ring_count; i++) {
     struct buf_ring_descriptor *desc = machine->buffer_rings + i;
     io_uring_free_buf_ring(&machine->ring, desc->br, desc->buf_count, i);
@@ -59,8 +69,7 @@ inline struct io_uring_sqe *um_get_sqe(struct um *machine, struct um_op *op) {
   sqe = io_uring_get_sqe(&machine->ring);
   if (likely(sqe)) goto done;
 
-  fprintf(stderr, "!!!Failed to get SQE\n");
-  um_raise_internal_error("Failed to get SQE");
+  um_raise_internal_error("Submission queue full. Consider raising the machine size.");
 
   // TODO: retry getting SQE?
 
@@ -271,37 +280,60 @@ inline void um_profile_wait_cqe_post(struct um *machine, double time_monotonic0,
   // machine->metrics.time_last_cpu = time_cpu;
 }
 
+inline void *um_wait_for_sidecar_signal(void *ptr) {
+  struct um *machine = ptr;
+  um_sidecar_signal_wait(machine);
+  return NULL;
+}
+
 // Waits for the given minimum number of completion entries. The wait_nr is
 // either 1 - where we wait for at least one CQE to be ready, or 0, where we
 // don't wait, and just process any CQEs that already ready.
 static inline void um_wait_for_and_process_ready_cqes(struct um *machine, int wait_nr) {
   struct wait_for_cqe_ctx ctx = { .machine = machine, .cqe = NULL, .wait_nr = wait_nr };
   machine->metrics.total_waits++;
-  double time_monotonic0 = 0.0;
-  VALUE fiber;
-  if (machine->profile_mode) um_profile_wait_cqe_pre(machine, &time_monotonic0, &fiber);
-  rb_thread_call_without_gvl(um_wait_for_cqe_without_gvl, (void *)&ctx, RUBY_UBF_IO, 0);
-  if (machine->profile_mode) um_profile_wait_cqe_post(machine, time_monotonic0, fiber);
-
-  if (unlikely(ctx.result < 0)) {
-    // the internal calls to (maybe submit) and wait for cqes may fail with:
-    // -EINTR (interrupted by signal)
-    // -EAGAIN (apparently can be returned when wait_nr = 0)
-    // both should not raise an exception.
-    switch (ctx.result) {
-      case -EINTR:
-      case -EAGAIN:
-        // do nothing
-        break;
-      default:
-        rb_syserr_fail(-ctx.result, strerror(-ctx.result));
-    }
-  }
 
-  if (ctx.cqe) {
-    um_process_cqe(machine, ctx.cqe);
-    io_uring_cq_advance(&machine->ring, 1);
+  if (machine->sidecar_mode) {
+    // fprintf(stderr, ">> sidecar wait cqes (unsubmitted: %d)\n", machine->metrics.ops_unsubmitted);
+    if (machine->metrics.ops_unsubmitted) {
+      io_uring_submit(&machine->ring);
+      machine->metrics.ops_unsubmitted = 0;
+    }
+    if (wait_nr) {
+      // fprintf(stderr, ">> um_wait_for_sidecar_signal\n");
+      // rb_thread_call_without_gvl(um_wait_for_sidecar_signal, (void *)machine, RUBY_UBF_PROCESS, 0);
+      // fprintf(stderr, "<< um_wait_for_sidecar_signal\n");
+      um_sidecar_signal_wait(machine);
+    }
     um_process_ready_cqes(machine);
+    // fprintf(stderr, "<< sidecar wait cqes\n");
+  }
+  else {
+    double time_monotonic0 = 0.0;
+    VALUE fiber;
+    if (machine->profile_mode) um_profile_wait_cqe_pre(machine, &time_monotonic0, &fiber);
+    rb_thread_call_without_gvl(um_wait_for_cqe_without_gvl, (void *)&ctx, RUBY_UBF_IO, 0);
+    if (machine->profile_mode) um_profile_wait_cqe_post(machine, time_monotonic0, fiber);
+
+    if (unlikely(ctx.result < 0)) {
+      // the internal calls to (maybe submit) and wait for cqes may fail with:
+      // -EINTR (interrupted by signal)
+      // -EAGAIN (apparently can be returned when wait_nr = 0)
+      // both should not raise an exception.
+      switch (ctx.result) {
+        case -EINTR:
+        case -EAGAIN:
+          // do nothing
+          break;
+        default:
+          rb_syserr_fail(-ctx.result, strerror(-ctx.result));
+      }
+    }
+    if (ctx.cqe) {
+      um_process_cqe(machine, ctx.cqe);
+      io_uring_cq_advance(&machine->ring, 1);
+      um_process_ready_cqes(machine);
+    }
   }
 }
 
@@ -336,30 +368,15 @@ inline VALUE um_switch(struct um *machine) {
     &machine->ring, machine->metrics.ops_unsubmitted, machine->metrics.ops_pending,
     machine->metrics.total_ops
   );
+
   while (true) {
     struct um_op *op = um_runqueue_shift(machine);
     if (op) {
       if (unlikely(op->flags & OP_F_RUNQUEUE_SKIP)) continue;
 
-      // in case of a snooze, we need to prevent a situation where completions
-      // are not processed because the runqueue is never empty. Theoretically,
-      // we can still have a situation where multiple fibers are all doing a
-      // snooze repeatedly, which can prevent completions from being processed.
-
-      // is the op a snooze op and is this the same fiber as the current one?
-      if (unlikely(op->kind == OP_SCHEDULE && op->fiber == rb_fiber_current())) {
-        //  are there any pending ops (i.e. waiting for completion)?
-        if (machine->metrics.ops_pending > 0) {
-          // if yes, process completions, get runqueue head, put original op
-          // back on runqueue.
-          // um_process_ready_cqes(machine);
-          um_wait_for_and_process_ready_cqes(machine, 0);
-          struct um_op *op2 = um_runqueue_shift(machine);
-          if (likely(op2 && op2 != op)) {
-            um_runqueue_push(machine, op);
-            op = op2;
-          }
-        }
+      // in test mode we want to process I/O on each snooze
+      if (unlikely(machine->test_mode && (op->kind == OP_SCHEDULE))) {
+        um_wait_for_and_process_ready_cqes(machine, 0);
       }
       return process_runqueue_op(machine, op);
     }
@@ -592,10 +609,10 @@ VALUE um_writev(struct um *machine, int fd, int argc, VALUE *argv) {
 
     int completed = um_op_completed_p(&op);
     if (unlikely(!completed)) goto cancelled;
-  
+
     writev_res = op.result.res;
     if (unlikely(writev_res < 0)) goto done;
-      
+
     len -= writev_res;
     if (len) {
       um_advance_iovecs_for_writing(&iovecs_ptr, &iovecs_len, (size_t)writev_res);
@@ -1016,7 +1033,7 @@ VALUE um_select(struct um *machine, VALUE rfds, VALUE wfds, VALUE efds) {
   }
 
   while (pending) {
-    um_wait_for_and_process_ready_cqes(machine, 0);
+    um_wait_for_and_process_ready_cqes(machine, 1);
 
     for (uint i = 0; i < total_len; i++) {
       struct um_op *op = ops + i;

data/ext/um/um.h

@@ -156,7 +156,11 @@ struct um {
   uint mark; // used to mark instances for debugging
 
   struct um_metrics metrics;
+  int test_mode;
   int profile_mode;
+  int sidecar_mode;
+  pthread_t sidecar_thread;
+  uint32_t *sidecar_signal;
 
   uint buffer_ring_count; // number of registered buffer rings
 
@@ -227,7 +231,7 @@ extern VALUE cAsyncOp;
 extern VALUE eStreamRESPError;
 
 struct um *um_get_machine(VALUE self);
-void um_setup(VALUE self, struct um *machine, uint size, uint sqpoll_timeout_msec);
+void um_setup(VALUE self, struct um *machine, uint size, uint sqpoll_timeout_msec, int sidecar_mode);
 void um_teardown(struct um *machine);
 
 VALUE um_metrics(struct um *machine, struct um_metrics *metrics);
@@ -350,6 +354,7 @@ VALUE stream_get_line(struct um_stream *stream, VALUE buf, ssize_t maxlen);
 VALUE stream_get_string(struct um_stream *stream, VALUE buf, ssize_t len);
 VALUE resp_decode(struct um_stream *stream, VALUE out_buffer);
 void resp_encode(struct um_write_buffer *buf, VALUE obj);
+void resp_encode_cmd(struct um_write_buffer *buf, int argc, VALUE *argv);
 
 __attribute__((noreturn)) void um_raise_internal_error(const char *msg);
 
@@ -358,4 +363,9 @@ void write_buffer_update_len(struct um_write_buffer *buf);
 
 void um_define_net_constants(VALUE mod);
 
+void um_sidecar_setup(struct um *machine);
+void um_sidecar_teardown(struct um *machine);
+void um_sidecar_signal_wait(struct um *machine);
+void um_sidecar_signal_wake(struct um *machine);
+
 #endif // UM_H

data/ext/um/um_class.c

@@ -72,6 +72,7 @@ inline struct um *um_get_machine(VALUE self) {
 static inline uint get_sqpoll_timeout_msec(VALUE sqpoll_timeout) {
   switch (TYPE(sqpoll_timeout)) {
     case T_NIL:
+    case T_UNDEF:
     case T_FALSE:
       return 0;
     case T_FLOAT:
@@ -86,16 +87,24 @@ static inline uint get_sqpoll_timeout_msec(VALUE sqpoll_timeout) {
 }
 
 VALUE UM_initialize(int argc, VALUE *argv, VALUE self) {
+  static ID kwargs_ids[3];
   struct um *machine = RTYPEDDATA_DATA(self);
-  VALUE entries;
-  VALUE sqpoll_timeout;
-  rb_scan_args(argc, argv, "02", &entries, &sqpoll_timeout);
+  VALUE opts, kwargs[3] = {Qnil, Qnil, Qnil};
 
-  uint entries_i = NIL_P(entries) ? 0 : NUM2UINT(entries);
-  uint sqpoll_timeout_msec = get_sqpoll_timeout_msec(sqpoll_timeout);
+  if (!kwargs_ids[0]) {
+    kwargs_ids[0] = rb_intern_const("size");
+    kwargs_ids[1] = rb_intern_const("sqpoll");
+    kwargs_ids[2] = rb_intern_const("sidecar");
+  }
+  rb_scan_args(argc, argv, "0:", &opts);
+  if (!NIL_P(opts)) {
+    rb_get_kwargs(opts, kwargs_ids, 0, 3, kwargs);
+  }
 
+  uint entries_i = TYPE(kwargs[0]) == T_FIXNUM ? NUM2UINT(kwargs[0]) : 0;
+  uint sqpoll_timeout_msec = get_sqpoll_timeout_msec(kwargs[1]);
+  um_setup(self, machine, entries_i, sqpoll_timeout_msec, RTEST(kwargs[2]));
 
-  um_setup(self, machine, entries_i, sqpoll_timeout_msec);
   return self;
 }
 
@@ -121,12 +130,17 @@ VALUE UM_metrics(VALUE self) {
   return um_metrics(machine, &machine->metrics);
 }
 
-VALUE UM_profile_p(VALUE self) {
+VALUE UM_profile_mode_p(VALUE self) {
   struct um *machine = um_get_machine(self);
   return machine->profile_mode ? Qtrue : Qfalse;
 }
 
-VALUE UM_profile_set(VALUE self, VALUE value) {
+VALUE UM_sqpoll_mode_p(VALUE self) {
+  struct um *machine = um_get_machine(self);
+  return machine->sqpoll_mode ? Qtrue : Qfalse;
+}
+
+VALUE UM_profile_mode_set(VALUE self, VALUE value) {
   struct um *machine = um_get_machine(self);
   machine->profile_mode = RTEST(value);
   if (machine->profile_mode) {
@@ -136,6 +150,29 @@ VALUE UM_profile_set(VALUE self, VALUE value) {
   return value;
 }
 
+VALUE UM_test_mode_set(VALUE self, VALUE value) {
+  struct um *machine = um_get_machine(self);
+  machine->test_mode = RTEST(value);
+  return value;
+}
+
+VALUE UM_sidecar_mode_p(VALUE self) {
+  struct um *machine = um_get_machine(self);
+  return machine->sidecar_mode ? Qtrue : Qfalse;
+}
+
+VALUE UM_sidecar_start(VALUE self) {
+  struct um *machine = um_get_machine(self);
+  um_sidecar_setup(machine);
+  return self;
+}
+
+VALUE UM_sidecar_stop(VALUE self) {
+  struct um *machine = um_get_machine(self);
+  um_sidecar_teardown(machine);
+  return self;
+}
+
 VALUE UM_snooze(VALUE self) {
   struct um *machine = um_get_machine(self);
   um_schedule(machine, rb_fiber_current(), Qnil);
@@ -581,8 +618,14 @@ void Init_UM(void) {
   rb_define_method(cUM, "size", UM_size, 0);
   rb_define_method(cUM, "mark", UM_mark_m, 1);
   rb_define_method(cUM, "metrics", UM_metrics, 0);
-  rb_define_method(cUM, "profile?", UM_profile_p, 0);
-  rb_define_method(cUM, "profile", UM_profile_set, 1);
+  rb_define_method(cUM, "sqpoll_mode?", UM_sqpoll_mode_p, 0);
+  rb_define_method(cUM, "profile_mode?", UM_profile_mode_p, 0);
+  rb_define_method(cUM, "profile_mode=", UM_profile_mode_set, 1);
+  rb_define_method(cUM, "test_mode=", UM_test_mode_set, 1);
+
+  rb_define_method(cUM, "sidecar_mode?", UM_sidecar_mode_p, 0);
+  rb_define_method(cUM, "sidecar_start", UM_sidecar_start, 0);
+  rb_define_method(cUM, "sidecar_stop", UM_sidecar_stop, 0);
 
   rb_define_method(cUM, "setup_buffer_ring", UM_setup_buffer_ring, 2);
 
@@ -639,7 +682,7 @@ void Init_UM(void) {
   #ifdef HAVE_IO_URING_SEND_VECTORIZED
   rb_define_method(cUM, "sendv", UM_sendv, -1);
   #endif
-  
+
   rb_define_method(cUM, "send_bundle", UM_send_bundle, -1);
   rb_define_method(cUM, "setsockopt", UM_setsockopt, 4);
   rb_define_method(cUM, "socket", UM_socket, 4);

data/ext/um/um_sidecar.c

@@ -0,0 +1,106 @@
+#include "um.h"
+#include <stdatomic.h>
+#include <linux/futex.h>
+#include <sys/syscall.h>
+#include <pthread.h>
+#include <unistd.h>
+#include <ruby/thread.h>
+
+#define FUTEX2_SIZE_U32		        0x02
+#define SIDECAR_THREAD_STACK_SIZE PTHREAD_STACK_MIN
+
+#define RAISE_ON_ERR(ret) if (ret) rb_syserr_fail(errno, strerror(errno))
+
+static inline int futex(uint32_t *uaddr, int op, uint32_t val, const struct timespec *timeout, uint32_t *uaddr2, uint32_t val3) {
+  return syscall(SYS_futex, uaddr, op, val, timeout, uaddr2, val3);
+}
+
+struct futex_wait_ctx {
+  uint32_t *futexp;
+  uint32_t oldval;
+};
+
+void *futex_wait_without_gvl(void *ptr) {
+  struct futex_wait_ctx *ctx = ptr;
+  futex(ctx->futexp, FUTEX_WAIT, ctx->oldval, NULL, NULL, 0);
+  return NULL;
+}
+
+static inline void xchg_futex_wait(uint32_t *futexp, uint32_t oldval, uint32_t newval) {
+  struct futex_wait_ctx ctx = { futexp, oldval };
+  while (1) {
+    if (atomic_compare_exchange_strong(futexp, &newval, oldval))
+      break;
+
+    rb_thread_call_without_gvl(futex_wait_without_gvl, (void *)&ctx, RUBY_UBF_IO, 0);
+    // int ret = futex(futexp, FUTEX_WAIT, oldval, NULL, NULL, 0);
+
+  }
+}
+
+static inline void xchg_futex_wake(uint32_t *futexp, uint32_t oldval, uint32_t newval) {
+  while (1) {
+    if (atomic_compare_exchange_strong(futexp, &newval, oldval))
+      break;
+
+    usleep(1);
+  }
+
+  futex(futexp, FUTEX_WAKE, 1, NULL, NULL, 0);
+}
+
+inline void um_sidecar_signal_wait(struct um *machine) {
+  // wait for machine->sidecar_signal to equal 1, then reset it to 0
+  xchg_futex_wait(machine->sidecar_signal, 0, 1);
+}
+
+inline void um_sidecar_signal_wake(struct um *machine) {
+  // busy-wait for machine->sidecar_signal to equal 0, then set it to 1 and wakeup futex waiter
+  xchg_futex_wake(machine->sidecar_signal, 1, 0);
+}
+
+static void *sidecar_start(void *arg) {
+  pthread_setcanceltype(PTHREAD_CANCEL_ASYNCHRONOUS, NULL);
+  struct um *machine = arg;
+  while (1) {
+    int ret = io_uring_enter2(machine->ring.enter_ring_fd, 0, 1, IORING_ENTER_GETEVENTS, NULL, 0);
+    if (!ret) {
+      um_sidecar_signal_wake(machine);
+  }
+  }
+  return NULL;
+}
+
+void um_sidecar_setup(struct um *machine) {
+  if (machine->sidecar_thread) return;
+
+  int                 ret;
+  pthread_attr_t      attr;
+
+  ret = pthread_attr_init(&attr);
+  RAISE_ON_ERR(ret);
+
+  ret = pthread_attr_setstacksize(&attr, SIDECAR_THREAD_STACK_SIZE);
+  RAISE_ON_ERR(ret);
+
+  sigset_t sigmask;
+  sigemptyset(&sigmask);
+  ret = pthread_attr_setsigmask_np(&attr, &sigmask);
+  RAISE_ON_ERR(ret);
+
+  ret = pthread_create(&machine->sidecar_thread, &attr, sidecar_start, machine);
+  RAISE_ON_ERR(ret);
+
+  ret = pthread_attr_destroy(&attr);
+  RAISE_ON_ERR(ret);
+}
+
+
+void um_sidecar_teardown(struct um *machine) {
+  if (machine->sidecar_thread) {
+    pthread_cancel(machine->sidecar_thread);
+    pthread_join(machine->sidecar_thread, NULL);
+
+    machine->sidecar_thread = 0;
+  }
+}

data/ext/um/um_stream.c

@@ -393,3 +393,34 @@ void resp_encode(struct um_write_buffer *buf, VALUE obj) {
       um_raise_internal_error("Can't encode object");
   }
 }
+
+void resp_encode_cmd(struct um_write_buffer *buf, int argc, VALUE *argv) {
+  char tmp1[48];
+  char tmp2[60];
+
+  sprintf(tmp1, "*%d\r\n", argc);
+  write_buffer_append_cstr(buf, tmp1);
+  for (int i = 0; i < argc; i++) {
+    switch (TYPE(argv[i])) {
+      case T_FIXNUM:
+        sprintf(tmp1, "%ld", NUM2LONG(argv[i]));
+        sprintf(tmp2, "$%ld\r\n%s\r\n", strlen(tmp1), (char *)tmp1);
+        write_buffer_append_cstr(buf, tmp2);
+        break;
+      case T_FLOAT:
+        sprintf(tmp1, "%lg", NUM2DBL(argv[i]));
+        sprintf(tmp2, "$%ld\r\n%s\r\n", strlen(tmp1), (char *)tmp1);
+        write_buffer_append_cstr(buf, tmp2);
+        break;
+      case T_STRING:
+        write_buffer_append_resp_bulk_string(buf, argv[i]);
+        break;
+      case T_SYMBOL:
+        write_buffer_append_resp_bulk_string(buf, rb_sym_to_s(argv[i]));
+        break;
+      default:
+        um_raise_internal_error("Can't encode object");
+    }
+  }
+  return;
+}

data/ext/um/um_stream_class.c

@@ -84,6 +84,18 @@ VALUE Stream_resp_encode(VALUE self, VALUE str, VALUE obj) {
   return str;
 }
 
+VALUE Stream_resp_encode_cmd(int argc, VALUE *argv, VALUE self) {
+  struct um_write_buffer buf;
+  VALUE str;
+  rb_check_arity(argc, 2, UNLIMITED_ARGUMENTS);
+  str = argv[0];
+  write_buffer_init(&buf, str);
+  rb_str_modify(str);
+  resp_encode_cmd(&buf, argc - 1, argv + 1);
+  write_buffer_update_len(&buf);
+  return str;
+}
+
 void Init_Stream(void) {
   VALUE cStream = rb_define_class_under(cUM, "Stream", rb_cObject);
   rb_define_alloc_func(cStream, Stream_allocate);
@@ -94,7 +106,9 @@ void Init_Stream(void) {
   rb_define_method(cStream, "get_string", Stream_get_string, 2);
 
   rb_define_method(cStream, "resp_decode", Stream_resp_decode, 0);
+
   rb_define_singleton_method(cStream, "resp_encode", Stream_resp_encode, 2);
+  rb_define_singleton_method(cStream, "resp_encode_cmd", Stream_resp_encode_cmd, -1);
 
   eStreamRESPError = rb_define_class_under(cStream, "RESPError", rb_eStandardError);
 }