uringmachine 0.11.1 → 0.12.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: ccb8488c3f313b7a3890ae110fc5cebc7ac0062bbaa9ef362fb7ff5b02c680b9
-  data.tar.gz: 41f2c23a9427609c4d01a6eef319cbd40d029cf85918fd6c8d00e5b502f83ef6
+  metadata.gz: 540e2e4df0b1953f58e36ab4dea2024a793b1bea4fd2358f64733f003b192510
+  data.tar.gz: 463cf782db4604358e1f54270e24e4bb97ee92176cafdb36079fd9d4a5132fda
 SHA512:
-  metadata.gz: 9bd08ebae21a7a10407ff04102d9cbf491bfab65ebe601b6ab3a6b7790632c946873415af4f45321e959bd39e875ff386e8c85f74329743204a9c0c047ce5da0
-  data.tar.gz: 3cd63222c0b990ef83f2b324aa46b36bd142a7b43a1a273428a9b70a95cfe25e0a1b06a2985a7463fe51a995c78e82838e34ee20d3c332cc46278ffd738652dd
+  metadata.gz: 9e54c0f81a6a6a0523a5210a00e5bd7d3a7c0598d8e15c4434080429ae10328079a06d1075fab88894eb2c6ac827e8c929b4fbc18a2bdc7cf20d7efda0d1bd81
+  data.tar.gz: 9de7ecd460344ec496fc061edb15a9fadd8b9a5b81db0ca8844afd6cc81bb79b7881af54a008b3c43565725f9234ea66728e69c7eaa6a09d090a0977128e0326

data/CHANGELOG.md

@@ -1,3 +1,13 @@
+# 2025-06-07 Version 0.12.1
+
+- Improve portability of `UM` constants
+
+# 2025-06-03 Version 0.12
+
+- Add buffer, maxlen params to `Stream#get_line`
+- Add buffer param to `Stream#get_string`
+- Remove `Stream#resp_get_line`, `Stream#resp_get_string` methods
+
 # 2025-06-02 Version 0.11.1
 
 - Fix `UM::Stream` behaviour on GC

data/TODO.md

@@ -22,3 +22,27 @@
 When doing a `call`, we need to provide a mailbox for the response. can this be
 automatic?
 
+# streams
+
+We're still missing:
+
+- limit on line length in `get_line`
+- ability to supply buffer to `get_line` and `get_string`
+- allow read to eof, maybe with `read_to_eof`
+
+For the sake of performance, simplicity and explicitness, we change the API as follows:
+
+```ruby
+stream.get_line(buf, limit)
+# the defaults:
+stream.get_line(nil, -1)
+
+stream.get_string(len, buf)
+# defaults:
+stream.get_string(len, nil)
+
+# and
+stream.read_to_eof(buf)
+# defaults:
+stream.read_to_eof(nil)
+```

data/examples/bm_http_parse.rb

@@ -132,11 +132,12 @@ end
 def stream_parse_headers(fd)
   stream = UM::Stream.new($machine, fd)
 
-  headers = stream_get_request_line(stream)
+  buf = String.new(capacity: 65536)
+  headers = stream_get_request_line(stream, buf)
   return nil if !headers
 
   while true
-    line = stream.get_line()
+    line = stream.get_line(buf, 0)
     break if line.empty?
 
     m = line.match(RE_HEADER_LINE)
@@ -148,8 +149,8 @@ def stream_parse_headers(fd)
   headers
 end
 
-def stream_get_request_line(stream)
-  line = stream.get_line()
+def stream_get_request_line(stream, buf)
+  line = stream.get_line(buf, 0)
 
   m = line.match(RE_REQUEST_LINE)
   return nil if !m
@@ -181,42 +182,43 @@ ensure
   ($machine.close(wfd) rescue nil) if wfd
 end
 
-# 10000.times { parse_http_parser }
-# 10000.times { parse_http_stringio }
-# 10000.times { parse_http_stream }
-# exit
-
-# GC.disable
-
-# OS = ObjectSpace
-
-# def object_count
-#   counts = ObjectSpace.count_objects
-#   counts[:TOTAL] - counts[:FREE]
-# end
+def compare_allocs
+  GC.disable
+  x = 1000
+  p(
+    alloc_http_parser:  alloc_count { x.times { parse_http_parser } },
+    alloc_stringio:     alloc_count { x.times { parse_http_stringio } },
+    alloc_stream:       alloc_count { x.times { parse_http_stream } }
+  )
+ensure
+  GC.enable
+end
 
-# def alloc_count
-#   GC.start
-#   count0 = object_count
-#   yield
-#   count1 = object_count
-#   count1 - count0
-# end
+def object_count
+  counts = ObjectSpace.count_objects
+  counts[:TOTAL] - counts[:FREE]
+end
 
-# X = 100
-# p(
-#   alloc_http_parser: alloc_count { X.times { parse_http_parser } },
-#   alloc_stringio: alloc_count { X.times { parse_http_stringio } },
-#   alloc_stream: alloc_count { X.times { parse_http_stream } }
-# )
-# exit
+def alloc_count
+  GC.start
+  count0 = object_count
+  yield
+  # GC.start
+  count1 = object_count
+  count1 - count0
+end
 
-Benchmark.ips do |x|
-  x.config(:time => 5, :warmup => 3)
+def benchmark
+  Benchmark.ips do |x|
+    x.config(:time => 5, :warmup => 3)
 
-  x.report("http_parser") { parse_http_parser }
-  x.report("stringio") { parse_http_stringio }
-  x.report("stream") { parse_http_stream }
+    x.report("http_parser") { parse_http_parser }
+    x.report("stringio") { parse_http_stringio }
+    x.report("stream") { parse_http_stream }
 
-  x.compare!
+    x.compare!
+  end
 end
+
+compare_allocs
+benchmark

data/ext/um/um.c

@@ -285,7 +285,7 @@ struct op_ctx {
   int flags;
 };
 
-VALUE um_timeout_ensure(VALUE arg) {
+VALUE um_timeout_complete(VALUE arg) {
   struct op_ctx *ctx = (struct op_ctx *)arg;
 
   if (!um_op_completed_p(ctx->op)) {
@@ -311,7 +311,7 @@ VALUE um_timeout(struct um *machine, VALUE interval, VALUE class) {
   io_uring_prep_timeout(sqe, &op->ts, 0, 0);
 
   struct op_ctx ctx = { .machine = machine, .op = op };
-  return rb_ensure(rb_yield, Qnil, um_timeout_ensure, (VALUE)&ctx);
+  return rb_ensure(rb_yield, Qnil, um_timeout_complete, (VALUE)&ctx);
 }
 
 /*******************************************************************************
@@ -653,7 +653,7 @@ VALUE um_statx(struct um *machine, int dirfd, VALUE path, int flags, unsigned in
                             multishot ops
 *******************************************************************************/
 
-VALUE accept_each_begin(VALUE arg) {
+VALUE accept_each_start(VALUE arg) {
   struct op_ctx *ctx = (struct op_ctx *)arg;
   struct io_uring_sqe *sqe = um_get_sqe(ctx->machine, ctx->op);
   io_uring_prep_multishot_accept(sqe, ctx->fd, NULL, NULL, 0);
@@ -684,7 +684,7 @@ VALUE accept_each_begin(VALUE arg) {
   return Qnil;
 }
 
-VALUE multishot_ensure(VALUE arg) {
+VALUE multishot_complete(VALUE arg) {
   struct op_ctx *ctx = (struct op_ctx *)arg;
   if (ctx->op->multishot_result_count) {
     int more = ctx->op->multishot_result_tail->flags & IORING_CQE_F_MORE;
@@ -706,7 +706,7 @@ VALUE um_accept_each(struct um *machine, int fd) {
   um_prep_op(machine, &op, OP_ACCEPT_MULTISHOT);
 
   struct op_ctx ctx = { .machine = machine, .op = &op, .fd = fd, .read_buf = NULL };
-  return rb_ensure(accept_each_begin, (VALUE)&ctx, multishot_ensure, (VALUE)&ctx);
+  return rb_ensure(accept_each_start, (VALUE)&ctx, multishot_complete, (VALUE)&ctx);
 }
 
 int um_read_each_singleshot_loop(struct op_ctx *ctx) {
@@ -771,7 +771,7 @@ void read_recv_each_prep(struct io_uring_sqe *sqe, struct op_ctx *ctx) {
   }
 }
 
-VALUE read_recv_each_begin(VALUE arg) {
+VALUE read_recv_each_start(VALUE arg) {
   struct op_ctx *ctx = (struct op_ctx *)arg;
   struct io_uring_sqe *sqe = um_get_sqe(ctx->machine, ctx->op);
   read_recv_each_prep(sqe, ctx);
@@ -809,7 +809,7 @@ VALUE um_read_each(struct um *machine, int fd, int bgid) {
   um_prep_op(machine, &op, OP_READ_MULTISHOT);
 
   struct op_ctx ctx = { .machine = machine, .op = &op, .fd = fd, .bgid = bgid, .read_buf = NULL };
-  return rb_ensure(read_recv_each_begin, (VALUE)&ctx, multishot_ensure, (VALUE)&ctx);
+  return rb_ensure(read_recv_each_start, (VALUE)&ctx, multishot_complete, (VALUE)&ctx);
 }
 
 VALUE um_recv_each(struct um *machine, int fd, int bgid, int flags) {
@@ -817,10 +817,10 @@ VALUE um_recv_each(struct um *machine, int fd, int bgid, int flags) {
   um_prep_op(machine, &op, OP_RECV_MULTISHOT);
 
   struct op_ctx ctx = { .machine = machine, .op = &op, .fd = fd, .bgid = bgid, .read_buf = NULL, .flags = flags };
-  return rb_ensure(read_recv_each_begin, (VALUE)&ctx, multishot_ensure, (VALUE)&ctx);
+  return rb_ensure(read_recv_each_start, (VALUE)&ctx, multishot_complete, (VALUE)&ctx);
 }
 
-VALUE periodically_begin(VALUE arg) {
+VALUE periodically_start(VALUE arg) {
   struct op_ctx *ctx = (struct op_ctx *)arg;
   struct io_uring_sqe *sqe = um_get_sqe(ctx->machine, ctx->op);
   io_uring_prep_timeout(sqe, &ctx->ts, 0, IORING_TIMEOUT_MULTISHOT);
@@ -857,6 +857,6 @@ VALUE um_periodically(struct um *machine, double interval) {
   op.ts = um_double_to_timespec(interval);
 
   struct op_ctx ctx = { .machine = machine, .op = &op, .ts = op.ts, .read_buf = NULL };
-  return rb_ensure(periodically_begin, (VALUE)&ctx, multishot_ensure, (VALUE)&ctx);
+  return rb_ensure(periodically_start, (VALUE)&ctx, multishot_complete, (VALUE)&ctx);
 }
 

data/ext/um/um.h

@@ -124,7 +124,6 @@ struct um {
 };
 
 struct um_mutex {
-  VALUE self;
   uint32_t state;
 };
 
@@ -147,15 +146,11 @@ struct um_queue {
 };
 
 struct um_async_op {
-  VALUE self;
-
   struct um *machine;
   struct um_op *op;
 };
 
 struct um_stream {
-  VALUE self;
-
   struct um *machine;
   int fd;
   VALUE buffer;
@@ -259,7 +254,7 @@ struct um_mutex *Mutex_data(VALUE self);
 struct um_queue *Queue_data(VALUE self);
 
 void um_mutex_init(struct um_mutex *mutex);
-VALUE um_mutex_synchronize(struct um *machine, uint32_t *state);
+VALUE um_mutex_synchronize(struct um *machine, VALUE mutex, uint32_t *state);
 
 void um_queue_init(struct um_queue *queue);
 void um_queue_free(struct um_queue *queue);
@@ -270,11 +265,8 @@ VALUE um_queue_pop(struct um *machine, struct um_queue *queue);
 VALUE um_queue_unshift(struct um *machine, struct um_queue *queue, VALUE value);
 VALUE um_queue_shift(struct um *machine, struct um_queue *queue);
 
-int stream_read_more(struct um_stream *stream);
-VALUE stream_get_line(struct um_stream *stream);
-VALUE stream_get_string(struct um_stream *stream, ulong len);
-VALUE resp_get_line(struct um_stream *stream, VALUE out_buffer);
-VALUE resp_get_string(struct um_stream *stream, ulong len, VALUE out_buffer);
+VALUE stream_get_line(struct um_stream *stream, VALUE buf, ssize_t maxlen);
+VALUE stream_get_string(struct um_stream *stream, VALUE buf, ssize_t len);
 VALUE resp_decode(struct um_stream *stream, VALUE out_buffer);
 void resp_encode(struct um_write_buffer *buf, VALUE obj);
 

data/ext/um/um_async_op_class.c

@@ -7,44 +7,40 @@ VALUE SYM_timeout;
 
 static void AsyncOp_mark(void *ptr) {
   struct um_async_op *async_op = ptr;
-  rb_gc_mark_movable(async_op->self);
   rb_gc_mark_movable(async_op->machine->self);
 }
 
-static void AsyncOp_compact(void *ptr) {
-  struct um_async_op *async_op = ptr;
-  async_op->self = rb_gc_location(async_op->self);
-}
-
-static size_t AsyncOp_size(const void *ptr) {
-  return sizeof(struct um_async_op);
-}
-
 static void AsyncOp_free(void *ptr) {
   struct um_async_op *async_op = ptr;
-  um_op_free(async_op->machine, async_op->op);
-  free(ptr);
+  if (async_op->op)
+    um_op_free(async_op->machine, async_op->op);
 }
 
 static const rb_data_type_t AsyncOp_type = {
-    "UringMachine::AsyncOp",
-    {AsyncOp_mark, AsyncOp_free, AsyncOp_size, AsyncOp_compact},
-    0, 0, RUBY_TYPED_FREE_IMMEDIATELY | RUBY_TYPED_WB_PROTECTED
+  .wrap_struct_name = "UringMachine::AsyncOp",
+  .function = {
+    .dmark = AsyncOp_mark,
+    .dfree = AsyncOp_free,
+    .dsize = NULL,
+    .dcompact = NULL
+  },
+  .flags = RUBY_TYPED_FREE_IMMEDIATELY | RUBY_TYPED_WB_PROTECTED | RUBY_TYPED_EMBEDDABLE
 };
 
 static VALUE AsyncOp_allocate(VALUE klass) {
-  struct um_async_op *async_op = malloc(sizeof(struct um_async_op));
-  return TypedData_Wrap_Struct(klass, &AsyncOp_type, async_op);
+  struct um_async_op *async_op;
+  return TypedData_Make_Struct(klass, struct um_async_op, &AsyncOp_type, async_op);
 }
 
-inline struct um_async_op *AsyncOp_data(VALUE self) {
-  return RTYPEDDATA_DATA(self);
+static inline struct um_async_op *AsyncOp_data(VALUE self) {
+  struct um_async_op *async_op;
+  TypedData_Get_Struct(self, struct um_async_op, &AsyncOp_type, async_op);
+  return async_op;
 }
 
 VALUE AsyncOp_initialize(VALUE self) {
   struct um_async_op *async_op = AsyncOp_data(self);
   memset(async_op, 0, sizeof(struct um_async_op));
-  async_op->self = self;
   return self;
 }
 

data/ext/um/um_class.c

@@ -24,27 +24,28 @@ static void UM_free(void *ptr) {
   free(ptr);
 }
 
-static size_t UM_size(const void *ptr) {
-  return sizeof(struct um);
-}
-
-static const rb_data_type_t UM_type = {
-    "UringMachine",
-    {UM_mark, UM_free, UM_size, UM_compact},
-    0, 0, RUBY_TYPED_FREE_IMMEDIATELY | RUBY_TYPED_WB_PROTECTED
+static const rb_data_type_t UringMachine_type = {
+  .wrap_struct_name = "UringMachine",
+  .function = {
+    .dmark = UM_mark,
+    .dfree = UM_free,
+    .dsize = NULL,
+    .dcompact = UM_compact
+  },
+  .flags = RUBY_TYPED_FREE_IMMEDIATELY | RUBY_TYPED_WB_PROTECTED
 };
 
 static VALUE UM_allocate(VALUE klass) {
-  struct um *machine = ALLOC(struct um);
-
-  return TypedData_Wrap_Struct(klass, &UM_type, machine);
+  struct um *um;
+  return TypedData_Make_Struct(klass, struct um, &UringMachine_type, um);
 }
 
 inline struct um *um_get_machine(VALUE self) {
-  struct um *machine = RTYPEDDATA_DATA(self);
-  if (!machine->ring_initialized)
-    rb_raise(rb_eRuntimeError, "Machine not initialized");
-  return machine;
+  struct um *um;
+  TypedData_Get_Struct(self, struct um, &UringMachine_type, um);
+  if (!um->ring_initialized) rb_raise(rb_eRuntimeError, "Machine not initialized");
+  
+  return um;
 }
 
 VALUE UM_initialize(VALUE self) {
@@ -244,7 +245,7 @@ VALUE UM_setsockopt(VALUE self, VALUE fd, VALUE level, VALUE opt, VALUE value) {
 VALUE UM_mutex_synchronize(VALUE self, VALUE mutex) {
   struct um *machine = um_get_machine(self);
   struct um_mutex *mutex_data = Mutex_data(mutex);
-  return um_mutex_synchronize(machine, &mutex_data->state);
+  return um_mutex_synchronize(machine, mutex, &mutex_data->state);
 }
 
 VALUE UM_queue_push(VALUE self, VALUE queue, VALUE value) {
@@ -278,7 +279,7 @@ struct um_open_ctx {
   VALUE fd;
 };
 
-VALUE UM_open_ensure(VALUE arg) {
+VALUE UM_open_complete(VALUE arg) {
   struct um_open_ctx *ctx = (struct um_open_ctx *)arg;
   UM_close(ctx->self, ctx->fd);
   return ctx->self;
@@ -290,7 +291,7 @@ VALUE UM_open(VALUE self, VALUE pathname, VALUE flags) {
   VALUE fd = um_open(machine, pathname, NUM2INT(flags), 0666);
   if (rb_block_given_p()) {
     struct um_open_ctx ctx = { self, fd };
-    return rb_ensure(rb_yield, fd, UM_open_ensure, (VALUE)&ctx);
+    return rb_ensure(rb_yield, fd, UM_open_complete, (VALUE)&ctx);
   }
   else
     return fd;

data/ext/um/um_const.c

@@ -39,18 +39,24 @@ void um_define_net_constants(VALUE mod) {
   DEF_CONST_INT(mod, STATX_BASIC_STATS);
   DEF_CONST_INT(mod, STATX_BTIME);
   DEF_CONST_INT(mod, STATX_ALL);
+  #ifdef STATX_MNT_ID
   DEF_CONST_INT(mod, STATX_MNT_ID);
+  #endif
+  #ifdef STATX_DIOALIGN
   DEF_CONST_INT(mod, STATX_DIOALIGN);
+  #endif
+  #ifdef STATX_MNT_ID_UNIQUE
   DEF_CONST_INT(mod, STATX_MNT_ID_UNIQUE);
-#ifdef STATX_SUBVOL
+  #endif
+  #ifdef STATX_SUBVOL
   DEF_CONST_INT(mod, STATX_SUBVOL);
-#endif
-#ifdef STATX_WRITE_ATOMIC
+  #endif
+  #ifdef STATX_WRITE_ATOMIC
   DEF_CONST_INT(mod, STATX_WRITE_ATOMIC);
-#endif
-#ifdef STATX_DIO_READ_ALIGN
+  #endif
+  #ifdef STATX_DIO_READ_ALIGN
   DEF_CONST_INT(mod, STATX_DIO_READ_ALIGN);
-#endif
+  #endif
 
   DEF_CONST_INT(mod, MSG_CONFIRM);
   DEF_CONST_INT(mod, MSG_DONTROUTE);

data/ext/um/um_mutex_class.c

@@ -3,38 +3,30 @@
 
 VALUE cMutex;
 
-static void Mutex_mark(void *ptr) {
-  struct um_mutex *mutex = ptr;
-  rb_gc_mark_movable(mutex->self);
-}
-
-static void Mutex_compact(void *ptr) {
-  struct um_mutex *mutex = ptr;
-  mutex->self = rb_gc_location(mutex->self);
-}
-
-static size_t Mutex_size(const void *ptr) {
-  return sizeof(struct um_mutex);
-}
-
 static const rb_data_type_t Mutex_type = {
-    "UringMachineMutex",
-    {Mutex_mark, free, Mutex_size, Mutex_compact},
-    0, 0, RUBY_TYPED_FREE_IMMEDIATELY | RUBY_TYPED_WB_PROTECTED
+  .wrap_struct_name = "UringMachine::Mutex",
+  .function = {
+    .dmark = NULL,
+    .dfree = RUBY_TYPED_DEFAULT_FREE,
+    .dsize = NULL,
+    .dcompact = NULL
+  },
+  .flags = RUBY_TYPED_FREE_IMMEDIATELY | RUBY_TYPED_WB_PROTECTED | RUBY_TYPED_EMBEDDABLE
 };
 
 static VALUE Mutex_allocate(VALUE klass) {
-  struct um_mutex *mutex = malloc(sizeof(struct um_mutex));
-  return TypedData_Wrap_Struct(klass, &Mutex_type, mutex);
+  struct um_mutex *mutex;
+  return TypedData_Make_Struct(klass, struct um_mutex, &Mutex_type, mutex);
 }
 
 inline struct um_mutex *Mutex_data(VALUE self) {
-  return RTYPEDDATA_DATA(self);
+  struct um_mutex *mutex;
+  TypedData_Get_Struct(self, struct um_mutex, &Mutex_type, mutex);
+  return mutex;
 }
 
 VALUE Mutex_initialize(VALUE self) {
   struct um_mutex *mutex = Mutex_data(self);
-  mutex->self = self;
   um_mutex_init(mutex);
   return self;
 }

data/ext/um/um_queue_class.c

@@ -18,23 +18,26 @@ static void Queue_free(void *ptr) {
   um_queue_free(queue);
 }
 
-static size_t Queue_size(const void *ptr) {
-  return sizeof(struct um_queue);
-}
-
 static const rb_data_type_t Queue_type = {
-    "UringMachineQueue",
-    {Queue_mark, Queue_free, Queue_size, Queue_compact},
-    0, 0, RUBY_TYPED_FREE_IMMEDIATELY | RUBY_TYPED_WB_PROTECTED
+  .wrap_struct_name = "UringMachine::Queue",
+  .function = {
+    .dmark = Queue_mark,
+    .dfree = Queue_free,
+    .dsize = NULL,
+    .dcompact = Queue_compact
+  },
+  .flags = RUBY_TYPED_FREE_IMMEDIATELY | RUBY_TYPED_WB_PROTECTED | RUBY_TYPED_EMBEDDABLE
 };
 
 static VALUE Queue_allocate(VALUE klass) {
-  struct um_queue *queue = malloc(sizeof(struct um_queue));
-  return TypedData_Wrap_Struct(klass, &Queue_type, queue);
+  struct um_queue *queue;
+  return TypedData_Make_Struct(klass, struct um_queue, &Queue_type, queue);
 }
 
 inline struct um_queue *Queue_data(VALUE self) {
-  return RTYPEDDATA_DATA(self);
+  struct um_queue *queue;
+  TypedData_Get_Struct(self, struct um_queue, &Queue_type, queue);
+  return queue;
 }
 
 VALUE Queue_initialize(VALUE self) {

data/ext/um/um_stream.c

@@ -1,31 +1,5 @@
 #include "um.h"
-
-VALUE stream_get_line(struct um_stream *stream) {
-  char *start = RSTRING_PTR(stream->buffer) + stream->pos;
-  while (true) {
-    char * lf_ptr = memchr(start, '\n', stream->len - stream->pos);
-    if (lf_ptr) {
-      ulong len = lf_ptr - start;
-      if (len && (start[len - 1] == '\r')) len -= 1;
-
-      VALUE str = rb_str_new(start, len);
-      stream->pos += lf_ptr - start + 1;
-      return str;
-    }
-
-    if (!stream_read_more(stream)) return Qnil;
-  }
-}
-
-VALUE stream_get_string(struct um_stream *stream, ulong len) {
-  while (stream->len - stream->pos < len)
-    if (!stream_read_more(stream)) return Qnil;
-  
-  char *start = RSTRING_PTR(stream->buffer) + stream->pos;
-  VALUE str = rb_utf8_str_new(start, len);
-  stream->pos += len;
-  return str;
-}
+#include <stdlib.h>
 
 static inline void stream_check_truncate_buffer(struct um_stream *stream) {
   if ((stream->pos == stream->len) && (stream->len >= 1 << 12)) {
@@ -67,10 +41,67 @@ int stream_read_more(struct um_stream *stream) {
   return 1;
 }
 
-// ensure string can hold at least len bytes
+// ensures given string can hold at least given len bytes (+trailing null)
 static inline void str_expand(VALUE str, size_t len) {
-  size_t capa = rb_str_capacity(str);
-  if (capa < len + 1) rb_str_modify_expand(str, len + 1 - capa);
+  rb_str_resize(str, len);
+}
+
+static inline void str_copy_bytes(VALUE dest, const char *src, ssize_t len) {
+  str_expand(dest, len + 1);
+  char *dest_ptr = RSTRING_PTR(dest);
+  memcpy(dest_ptr, src, len);
+  dest_ptr[len] = 0;
+  rb_str_set_len(dest, len);
+}
+
+VALUE stream_get_line(struct um_stream *stream, VALUE buf, ssize_t maxlen) {
+  char *start = RSTRING_PTR(stream->buffer) + stream->pos;
+  while (true) {
+    ssize_t pending_len = stream->len - stream->pos;
+    ssize_t search_len = pending_len;
+    ssize_t absmax_len = labs(maxlen);
+    int should_limit_len = (absmax_len > 0) && (search_len > maxlen);
+    if (should_limit_len) search_len = absmax_len;
+
+    char * lf_ptr = memchr(start, '\n', search_len);
+    if (lf_ptr) {
+      ssize_t len = lf_ptr - start;
+      if (len && (start[len - 1] == '\r')) len -= 1;
+
+      stream->pos += lf_ptr - start + 1;
+      if (NIL_P(buf)) return rb_utf8_str_new(start, len);
+
+      str_copy_bytes(buf, start, len);
+      return buf;
+    }
+    else if (should_limit_len && pending_len > search_len)
+      // maxlen
+      return Qnil;
+
+    if (!stream_read_more(stream))
+      return Qnil;
+    else
+      // update start ptr (it might have changed after reading)
+      start = RSTRING_PTR(stream->buffer) + stream->pos;
+  }
+}
+
+VALUE stream_get_string(struct um_stream *stream, VALUE buf, ssize_t len) {
+  size_t abslen = labs(len);
+  while (stream->len - stream->pos < abslen)
+    if (!stream_read_more(stream)) {
+      if (len > 0) return Qnil;
+
+      abslen = stream->len - stream->pos;
+    }
+  
+  char *start = RSTRING_PTR(stream->buffer) + stream->pos;
+  stream->pos += abslen;
+
+  if (NIL_P(buf)) return rb_utf8_str_new(start, abslen);
+
+  str_copy_bytes(buf, start, len);
+  return buf;
 }
 
 VALUE resp_get_line(struct um_stream *stream, VALUE out_buffer) {
@@ -89,11 +120,7 @@ VALUE resp_get_line(struct um_stream *stream, VALUE out_buffer) {
         return str;
       }
 
-      str_expand(out_buffer, len + 1);
-      char *dest_ptr = RSTRING_PTR(out_buffer);
-      memcpy(dest_ptr, start, len);
-      dest_ptr[len] = 0; // add null at end
-      rb_str_set_len(out_buffer, len);
+      str_copy_bytes(out_buffer, start, len);
       return out_buffer;
     }
 
@@ -116,11 +143,7 @@ VALUE resp_get_string(struct um_stream *stream, ulong len, VALUE out_buffer) {
 
   if (NIL_P(out_buffer)) return rb_utf8_str_new(start, len);
 
-  str_expand(out_buffer, len + 1);
-  char *dest_ptr = RSTRING_PTR(out_buffer);
-  memcpy(dest_ptr, start, len);
-  dest_ptr[len] = 0; // add null at end
-  rb_str_set_len(out_buffer, len);
+  str_copy_bytes(out_buffer, start, len);
   return out_buffer;
 }
 

data/ext/um/um_stream_class.c

@@ -4,42 +4,40 @@ VALUE cStream;
 
 static void Stream_mark(void *ptr) {
   struct um_stream *stream = ptr;
-  rb_gc_mark_movable(stream->self);
   rb_gc_mark_movable(stream->buffer);
 }
 
 static void Stream_compact(void *ptr) {
   struct um_stream *stream = ptr;
-  stream->self = rb_gc_location(stream->self);
   stream->buffer = rb_gc_location(stream->buffer);
 }
 
-static void Stream_free(void *ptr) {
-  free(ptr);
-}
-
-static size_t Stream_size(const void *ptr) {
-  return sizeof(struct um_stream);
-}
-
 static const rb_data_type_t Stream_type = {
-    "UringMachine::Stream",
-    {Stream_mark, Stream_free, Stream_size, Stream_compact},
-    0, 0, RUBY_TYPED_FREE_IMMEDIATELY | RUBY_TYPED_WB_PROTECTED
+  .wrap_struct_name = "UringMachine::Stream",
+  .function = {
+    .dmark = Stream_mark,
+    .dfree = RUBY_TYPED_DEFAULT_FREE,
+    .dsize = NULL,
+    .dcompact = Stream_compact
+  },
+  .flags = RUBY_TYPED_FREE_IMMEDIATELY | RUBY_TYPED_WB_PROTECTED | RUBY_TYPED_EMBEDDABLE
 };
 
 static VALUE Stream_allocate(VALUE klass) {
-  struct um_stream *stream = ALLOC(struct um_stream);
+  struct um_stream *stream;
+  return TypedData_Make_Struct(klass, struct um_stream, &Stream_type, stream);
+}
 
-  return TypedData_Wrap_Struct(klass, &Stream_type, stream);
+static inline struct um_stream *Stream_data(VALUE self) {
+  struct um_stream *stream;
+  TypedData_Get_Struct(self, struct um_stream, &Stream_type, stream);
+  return stream;
 }
 
 VALUE Stream_initialize(VALUE self, VALUE machine, VALUE fd) {
-  struct um_stream *stream = RTYPEDDATA_DATA(self);
+  struct um_stream *stream = Stream_data(self);
 
-  stream->self = self;
-  
-  stream->machine = RTYPEDDATA_DATA(machine);
+  stream->machine = um_get_machine(machine);
   stream->fd = NUM2ULONG(fd);
   stream->buffer = rb_utf8_str_new_literal("");
   rb_str_resize(stream->buffer, 1 << 16); // 64KB
@@ -52,40 +50,22 @@ VALUE Stream_initialize(VALUE self, VALUE machine, VALUE fd) {
   return self;
 }
 
-VALUE Stream_get_line(VALUE self) {
-  struct um_stream *stream = RTYPEDDATA_DATA(self);
-  if (unlikely(stream->eof)) return Qnil;
-
-  return stream_get_line(stream);
-}
-
-VALUE Stream_get_string(VALUE self, VALUE len) {
-  struct um_stream *stream = RTYPEDDATA_DATA(self);
-  if (unlikely(stream->eof)) return Qnil;
-
-  return stream_get_string(stream, NUM2ULONG(len));
-}
-
-VALUE Stream_resp_get_line(VALUE self) {
-  struct um_stream *stream = RTYPEDDATA_DATA(self);
+VALUE Stream_get_line(VALUE self, VALUE buf, VALUE limit) {
+  struct um_stream *stream = Stream_data(self);
   if (unlikely(stream->eof)) return Qnil;
 
-  VALUE line = resp_get_line(stream, Qnil);
-  RB_GC_GUARD(line);
-  return line;
+  return stream_get_line(stream, buf, NUM2LONG(limit));
 }
 
-VALUE Stream_resp_get_string(VALUE self, VALUE len) {
-  struct um_stream *stream = RTYPEDDATA_DATA(self);
+VALUE Stream_get_string(VALUE self, VALUE buf, VALUE len) {
+  struct um_stream *stream = Stream_data(self);
   if (unlikely(stream->eof)) return Qnil;
 
-  VALUE str = resp_get_string(stream, NUM2ULONG(len), Qnil);
-  RB_GC_GUARD(str);
-  return str;
+  return stream_get_string(stream, buf, NUM2LONG(len));
 }
 
 VALUE Stream_resp_decode(VALUE self) {
-  struct um_stream *stream = RTYPEDDATA_DATA(self);
+  struct um_stream *stream = Stream_data(self);
   if (unlikely(stream->eof)) return Qnil;
 
   VALUE out_buffer = rb_utf8_str_new_literal("");
@@ -109,11 +89,8 @@ void Init_Stream(void) {
 
   rb_define_method(cStream, "initialize", Stream_initialize, 2);
   
-  rb_define_method(cStream, "get_line", Stream_get_line, 0);
-  rb_define_method(cStream, "get_string", Stream_get_string, 1);
-
-  rb_define_method(cStream, "resp_get_line", Stream_resp_get_line, 0);
-  rb_define_method(cStream, "resp_get_string", Stream_resp_get_string, 1);
+  rb_define_method(cStream, "get_line", Stream_get_line, 2);
+  rb_define_method(cStream, "get_string", Stream_get_string, 2);
 
   rb_define_method(cStream, "resp_decode", Stream_resp_decode, 0);
   

data/ext/um/um_sync.c

@@ -73,24 +73,28 @@ inline void um_mutex_unlock(struct um *machine, uint32_t *state) {
 
 struct sync_ctx {
   struct um *machine;
+  VALUE mutex;
   uint32_t *state;
 };
 
-VALUE synchronize_begin(VALUE arg) {
+VALUE synchronize_start(VALUE arg) {
   struct sync_ctx *ctx = (struct sync_ctx *)arg;
   um_mutex_lock(ctx->machine, ctx->state);
   return rb_yield(Qnil);
 }
 
-VALUE synchronize_ensure(VALUE arg) {
+VALUE synchronize_complete(VALUE arg) {
   struct sync_ctx *ctx = (struct sync_ctx *)arg;
+  // Mutex is an embedded data class, so it might have moved while the operation
+  // was ongoing. We need to update the pointer to the embedded state variable.
+  ctx->state = &Mutex_data(ctx->mutex)->state;
   um_mutex_unlock(ctx->machine, ctx->state);
   return Qnil;
 }
 
-inline VALUE um_mutex_synchronize(struct um *machine, uint32_t *state) {
-  struct sync_ctx ctx = { .machine = machine, .state = state };
-  return rb_ensure(synchronize_begin, (VALUE)&ctx, synchronize_ensure, (VALUE)&ctx);
+inline VALUE um_mutex_synchronize(struct um *machine, VALUE mutex, uint32_t *state) {
+  struct sync_ctx ctx = { .machine = machine, .mutex = mutex, .state = state };
+  return rb_ensure(synchronize_start, (VALUE)&ctx, synchronize_complete, (VALUE)&ctx);
 }
 
 #define QUEUE_EMPTY 0
@@ -116,8 +120,6 @@ inline void um_queue_free(struct um_queue *queue) {
     free(entry);
     entry = next;
   }
-
-  free(queue);
 }
 
 inline void um_queue_mark(struct um_queue *queue) {
@@ -226,11 +228,12 @@ enum queue_op { QUEUE_POP, QUEUE_SHIFT };
 
 struct queue_wait_ctx {
   struct um *machine;
+  VALUE queue_obj;
   struct um_queue *queue;
   enum queue_op op;
 };
 
-VALUE um_queue_remove_begin(VALUE arg) {
+VALUE um_queue_remove_start(VALUE arg) {
   struct queue_wait_ctx *ctx = (struct queue_wait_ctx *)arg;
 
   ctx->queue->num_waiters++;
@@ -247,9 +250,13 @@ VALUE um_queue_remove_begin(VALUE arg) {
   return (ctx->op == QUEUE_POP ? queue_remove_tail : queue_remove_head)(ctx->queue);
 }
 
-VALUE um_queue_remove_ensure(VALUE arg) {
+VALUE um_queue_remove_complete(VALUE arg) {
   struct queue_wait_ctx *ctx = (struct queue_wait_ctx *)arg;
 
+  // the um_queue struct is embedded, so it might have been moved while the op
+  // was ongoing, so we need to get it again on op completion
+  ctx->queue = Queue_data(ctx->queue_obj);
+
   ctx->queue->num_waiters--;
 
   if (ctx->queue->num_waiters && ctx->queue->tail) {
@@ -263,11 +270,11 @@ VALUE um_queue_remove_ensure(VALUE arg) {
 }
 
 VALUE um_queue_pop(struct um *machine, struct um_queue *queue) {
-  struct queue_wait_ctx ctx = { .machine = machine, .queue = queue, .op = QUEUE_POP };
-  return rb_ensure(um_queue_remove_begin, (VALUE)&ctx, um_queue_remove_ensure, (VALUE)&ctx);
+  struct queue_wait_ctx ctx = { .machine = machine, .queue_obj = queue->self, .queue = queue, .op = QUEUE_POP };
+  return rb_ensure(um_queue_remove_start, (VALUE)&ctx, um_queue_remove_complete, (VALUE)&ctx);
 }
 
 VALUE um_queue_shift(struct um *machine, struct um_queue *queue) {
-  struct queue_wait_ctx ctx = { .machine = machine, .queue = queue, .op = QUEUE_SHIFT };
-  return rb_ensure(um_queue_remove_begin, (VALUE)&ctx, um_queue_remove_ensure, (VALUE)&ctx);
+  struct queue_wait_ctx ctx = { .machine = machine, .queue_obj = queue->self, .queue = queue, .op = QUEUE_SHIFT };
+  return rb_ensure(um_queue_remove_start, (VALUE)&ctx, um_queue_remove_complete, (VALUE)&ctx);
 }

data/lib/uringmachine/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 class UringMachine
-  VERSION = '0.11.1'
+  VERSION = '0.12.1'
 end

data/test/test_stream.rb

@@ -15,41 +15,92 @@ class StreamTest < StreamBaseTest
     machine.write(@wfd, "foo\nbar\r\nbaz")
     machine.close(@wfd)
 
-    assert_equal 'foo', @stream.get_line
-    assert_equal 'bar', @stream.get_line
-    assert_nil @stream.get_line
+    assert_equal 'foo', @stream.get_line(nil, 0)
+    assert_equal 'bar', @stream.get_line(nil, 0)
+    assert_nil @stream.get_line(nil, 0)
+  end
+
+  def test_get_line_with_buf
+    machine.write(@wfd, "foo\nbar\r\nbaz")
+    machine.close(@wfd)
+
+    buf = +''
+    ret = @stream.get_line(buf, 0)
+    assert_equal 'foo', buf
+    assert_equal ret, buf
+
+    ret = @stream.get_line(buf, 0)
+    assert_equal 'bar', buf
+    assert_equal ret, buf
+  end
+
+  def test_get_line_with_positive_maxlen
+    machine.write(@wfd, "foobar\r\n")
+    machine.close(@wfd)
+
+    buf = +''
+    ret = @stream.get_line(buf, 3)
+    assert_nil ret
+    assert_equal '', buf
+
+    # verify that stream pos has not changed
+    ret = @stream.get_line(buf, 0)
+    assert_equal 'foobar', buf
+    assert_equal ret, buf
+  end
+
+  def test_get_line_with_negative_maxlen
+    machine.write(@wfd, "foobar\r\n")
+    machine.close(@wfd)
+
+    buf = +''
+    ret = @stream.get_line(buf, -3)
+    assert_nil ret
+    assert_equal '', buf
+
+    # verify that stream pos has not changed
+    ret = @stream.get_line(buf, 0)
+    assert_equal 'foobar', buf
+    assert_equal ret, buf
   end
 
   def test_get_string
     machine.write(@wfd, "foobarbazblahzzz")
     machine.close(@wfd)
 
-    assert_equal 'foobar', @stream.get_string(6)
-    assert_equal 'baz', @stream.get_string(3)
-    assert_equal 'blah', @stream.get_string(4)
-    assert_nil @stream.get_string(4)
+    assert_equal 'foobar', @stream.get_string(nil, 6)
+    assert_equal 'baz', @stream.get_string(nil, 3)
+    assert_equal 'blah', @stream.get_string(nil, 4)
+    assert_nil @stream.get_string(nil, 4)
   end
-end
 
-class StreamRespTest < StreamBaseTest
-  def test_trdp_get_line
-    machine.write(@wfd, "foo\r\nbarbar\r\nbaz\n")
+  def test_get_string_with_buf
+    machine.write(@wfd, "foobarbazblahzzz")
     machine.close(@wfd)
-    
-    assert_equal 'foo', @stream.resp_get_line
-    assert_equal 'barbar', @stream.resp_get_line
-    assert_nil @stream.resp_get_line
+
+    buf = +''
+    ret = @stream.get_string(buf, 6)
+    assert_equal 'foobar', buf
+    assert_equal ret, buf
+
+    ret = @stream.get_string(buf, 3)
+    assert_equal 'baz', buf
+    assert_equal ret, buf
   end
 
-  def test_resp_get_string
-    machine.write(@wfd, "foo\r\nbarbar\r\nbaz\n")
+  def test_get_string_with_negative_len
+    machine.write(@wfd, "foobar")
     machine.close(@wfd)
 
-    assert_equal 'foo', @stream.resp_get_string(3)
-    assert_equal 'barbar', @stream.resp_get_string(6)
-    assert_nil @stream.resp_get_string(3)
+    ret = @stream.get_string(nil, -12)
+    assert_equal 'foobar', ret
+
+    ret = @stream.get_string(nil, -4)
+    assert_nil ret
   end
+end
 
+class StreamRespTest < StreamBaseTest
   def test_resp_decode
     machine.write(@wfd, "+foo bar\r\n")
     assert_equal "foo bar", @stream.resp_decode

data/test/test_um.rb

@@ -271,7 +271,7 @@ class PeriodicallyTest < UMBaseTest
     rescue Cancel
       cancel = 1
     end
-    machine.snooze
+    2.times { machine.snooze }
     assert_equal 0, machine.pending_count
     t1 = monotonic_clock
     assert_in_range 0.05..0.08, t1 - t0

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: uringmachine
 version: !ruby/object:Gem::Version
-  version: 0.11.1
+  version: 0.12.1
 platform: ruby
 authors:
 - Sharon Rosner