uringmachine 0.11.1 → 0.12

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: ccb8488c3f313b7a3890ae110fc5cebc7ac0062bbaa9ef362fb7ff5b02c680b9
-  data.tar.gz: 41f2c23a9427609c4d01a6eef319cbd40d029cf85918fd6c8d00e5b502f83ef6
+  metadata.gz: 1f8224da7817436bd74a52e650362c9b77d5dbc8877c0fa1cefd1c52e57ceb03
+  data.tar.gz: c7bb88c42bf59a9d1f105e2356c4f71f0fec49c7cbbe662744465ee3b34d5141
 SHA512:
-  metadata.gz: 9bd08ebae21a7a10407ff04102d9cbf491bfab65ebe601b6ab3a6b7790632c946873415af4f45321e959bd39e875ff386e8c85f74329743204a9c0c047ce5da0
-  data.tar.gz: 3cd63222c0b990ef83f2b324aa46b36bd142a7b43a1a273428a9b70a95cfe25e0a1b06a2985a7463fe51a995c78e82838e34ee20d3c332cc46278ffd738652dd
+  metadata.gz: 39d930e9f4d8a17676c28e4bf24f38b585dbd0cd124316272778abb285391be4bb9ecf14452d19568ffa8f6a0144c6a98b2e3ee1f50d58ec7c5101333f5de5e7
+  data.tar.gz: 95468f71d402a2a8abf675dbbae6e94b4e1919c5552f0b831b7387016d3fb24271756585cf66283130ed3440818ccc94f504d73a46d2ab8f9faa4607b0cc822b

data/CHANGELOG.md

@@ -1,3 +1,9 @@
+# 2025-06-03 Version 0.12
+
+- Add buffer, maxlen params to `Stream#get_line`
+- Add buffer param to `Stream#get_string`
+- Remove `Stream#resp_get_line`, `Stream#resp_get_string` methods
+
 # 2025-06-02 Version 0.11.1
 
 - Fix `UM::Stream` behaviour on GC

data/TODO.md

@@ -22,3 +22,27 @@
 When doing a `call`, we need to provide a mailbox for the response. can this be
 automatic?
 
+# streams
+
+We're still missing:
+
+- limit on line length in `get_line`
+- ability to supply buffer to `get_line` and `get_string`
+- allow read to eof, maybe with `read_to_eof`
+
+For the sake of performance, simplicity and explicitness, we change the API as follows:
+
+```ruby
+stream.get_line(buf, limit)
+# the defaults:
+stream.get_line(nil, -1)
+
+stream.get_string(len, buf)
+# defaults:
+stream.get_string(len, nil)
+
+# and
+stream.read_to_eof(buf)
+# defaults:
+stream.read_to_eof(nil)
+```

data/examples/bm_http_parse.rb

@@ -132,11 +132,12 @@ end
 def stream_parse_headers(fd)
   stream = UM::Stream.new($machine, fd)
 
-  headers = stream_get_request_line(stream)
+  buf = String.new(capacity: 65536)
+  headers = stream_get_request_line(stream, buf)
   return nil if !headers
 
   while true
-    line = stream.get_line()
+    line = stream.get_line(buf, 0)
     break if line.empty?
 
     m = line.match(RE_HEADER_LINE)
@@ -148,8 +149,8 @@ def stream_parse_headers(fd)
   headers
 end
 
-def stream_get_request_line(stream)
-  line = stream.get_line()
+def stream_get_request_line(stream, buf)
+  line = stream.get_line(buf, 0)
 
   m = line.match(RE_REQUEST_LINE)
   return nil if !m
@@ -181,42 +182,43 @@ ensure
   ($machine.close(wfd) rescue nil) if wfd
 end
 
-# 10000.times { parse_http_parser }
-# 10000.times { parse_http_stringio }
-# 10000.times { parse_http_stream }
-# exit
-
-# GC.disable
-
-# OS = ObjectSpace
-
-# def object_count
-#   counts = ObjectSpace.count_objects
-#   counts[:TOTAL] - counts[:FREE]
-# end
+def compare_allocs
+  GC.disable
+  x = 1000
+  p(
+    alloc_http_parser:  alloc_count { x.times { parse_http_parser } },
+    alloc_stringio:     alloc_count { x.times { parse_http_stringio } },
+    alloc_stream:       alloc_count { x.times { parse_http_stream } }
+  )
+ensure
+  GC.enable
+end
 
-# def alloc_count
-#   GC.start
-#   count0 = object_count
-#   yield
-#   count1 = object_count
-#   count1 - count0
-# end
+def object_count
+  counts = ObjectSpace.count_objects
+  counts[:TOTAL] - counts[:FREE]
+end
 
-# X = 100
-# p(
-#   alloc_http_parser: alloc_count { X.times { parse_http_parser } },
-#   alloc_stringio: alloc_count { X.times { parse_http_stringio } },
-#   alloc_stream: alloc_count { X.times { parse_http_stream } }
-# )
-# exit
+def alloc_count
+  GC.start
+  count0 = object_count
+  yield
+  # GC.start
+  count1 = object_count
+  count1 - count0
+end
 
-Benchmark.ips do |x|
-  x.config(:time => 5, :warmup => 3)
+def benchmark
+  Benchmark.ips do |x|
+    x.config(:time => 5, :warmup => 3)
 
-  x.report("http_parser") { parse_http_parser }
-  x.report("stringio") { parse_http_stringio }
-  x.report("stream") { parse_http_stream }
+    x.report("http_parser") { parse_http_parser }
+    x.report("stringio") { parse_http_stringio }
+    x.report("stream") { parse_http_stream }
 
-  x.compare!
+    x.compare!
+  end
 end
+
+compare_allocs
+benchmark

data/ext/um/um.h

@@ -270,11 +270,8 @@ VALUE um_queue_pop(struct um *machine, struct um_queue *queue);
 VALUE um_queue_unshift(struct um *machine, struct um_queue *queue, VALUE value);
 VALUE um_queue_shift(struct um *machine, struct um_queue *queue);
 
-int stream_read_more(struct um_stream *stream);
-VALUE stream_get_line(struct um_stream *stream);
-VALUE stream_get_string(struct um_stream *stream, ulong len);
-VALUE resp_get_line(struct um_stream *stream, VALUE out_buffer);
-VALUE resp_get_string(struct um_stream *stream, ulong len, VALUE out_buffer);
+VALUE stream_get_line(struct um_stream *stream, VALUE buf, ssize_t maxlen);
+VALUE stream_get_string(struct um_stream *stream, VALUE buf, ssize_t len);
 VALUE resp_decode(struct um_stream *stream, VALUE out_buffer);
 void resp_encode(struct um_write_buffer *buf, VALUE obj);
 

data/ext/um/um_stream.c

@@ -1,31 +1,5 @@
 #include "um.h"
-
-VALUE stream_get_line(struct um_stream *stream) {
-  char *start = RSTRING_PTR(stream->buffer) + stream->pos;
-  while (true) {
-    char * lf_ptr = memchr(start, '\n', stream->len - stream->pos);
-    if (lf_ptr) {
-      ulong len = lf_ptr - start;
-      if (len && (start[len - 1] == '\r')) len -= 1;
-
-      VALUE str = rb_str_new(start, len);
-      stream->pos += lf_ptr - start + 1;
-      return str;
-    }
-
-    if (!stream_read_more(stream)) return Qnil;
-  }
-}
-
-VALUE stream_get_string(struct um_stream *stream, ulong len) {
-  while (stream->len - stream->pos < len)
-    if (!stream_read_more(stream)) return Qnil;
-  
-  char *start = RSTRING_PTR(stream->buffer) + stream->pos;
-  VALUE str = rb_utf8_str_new(start, len);
-  stream->pos += len;
-  return str;
-}
+#include <stdlib.h>
 
 static inline void stream_check_truncate_buffer(struct um_stream *stream) {
   if ((stream->pos == stream->len) && (stream->len >= 1 << 12)) {
@@ -67,10 +41,67 @@ int stream_read_more(struct um_stream *stream) {
   return 1;
 }
 
-// ensure string can hold at least len bytes
+// ensures given string can hold at least given len bytes (+trailing null)
 static inline void str_expand(VALUE str, size_t len) {
-  size_t capa = rb_str_capacity(str);
-  if (capa < len + 1) rb_str_modify_expand(str, len + 1 - capa);
+  rb_str_resize(str, len);
+}
+
+static inline void str_copy_bytes(VALUE dest, const char *src, ssize_t len) {
+  str_expand(dest, len + 1);
+  char *dest_ptr = RSTRING_PTR(dest);
+  memcpy(dest_ptr, src, len);
+  dest_ptr[len] = 0;
+  rb_str_set_len(dest, len);
+}
+
+VALUE stream_get_line(struct um_stream *stream, VALUE buf, ssize_t maxlen) {
+  char *start = RSTRING_PTR(stream->buffer) + stream->pos;
+  while (true) {
+    ssize_t pending_len = stream->len - stream->pos;
+    ssize_t search_len = pending_len;
+    ssize_t absmax_len = labs(maxlen);
+    int should_limit_len = (absmax_len > 0) && (search_len > maxlen);
+    if (should_limit_len) search_len = absmax_len;
+
+    char * lf_ptr = memchr(start, '\n', search_len);
+    if (lf_ptr) {
+      ssize_t len = lf_ptr - start;
+      if (len && (start[len - 1] == '\r')) len -= 1;
+
+      stream->pos += lf_ptr - start + 1;
+      if (NIL_P(buf)) return rb_utf8_str_new(start, len);
+
+      str_copy_bytes(buf, start, len);
+      return buf;
+    }
+    else if (should_limit_len && pending_len > search_len)
+      // maxlen
+      return Qnil;
+
+    if (!stream_read_more(stream))
+      return Qnil;
+    else
+      // update start ptr (it might have changed after reading)
+      start = RSTRING_PTR(stream->buffer) + stream->pos;
+  }
+}
+
+VALUE stream_get_string(struct um_stream *stream, VALUE buf, ssize_t len) {
+  size_t abslen = labs(len);
+  while (stream->len - stream->pos < abslen)
+    if (!stream_read_more(stream)) {
+      if (len > 0) return Qnil;
+
+      abslen = stream->len - stream->pos;
+    }
+  
+  char *start = RSTRING_PTR(stream->buffer) + stream->pos;
+  stream->pos += abslen;
+
+  if (NIL_P(buf)) return rb_utf8_str_new(start, abslen);
+
+  str_copy_bytes(buf, start, len);
+  return buf;
 }
 
 VALUE resp_get_line(struct um_stream *stream, VALUE out_buffer) {
@@ -89,11 +120,7 @@ VALUE resp_get_line(struct um_stream *stream, VALUE out_buffer) {
         return str;
       }
 
-      str_expand(out_buffer, len + 1);
-      char *dest_ptr = RSTRING_PTR(out_buffer);
-      memcpy(dest_ptr, start, len);
-      dest_ptr[len] = 0; // add null at end
-      rb_str_set_len(out_buffer, len);
+      str_copy_bytes(out_buffer, start, len);
       return out_buffer;
     }
 
@@ -116,11 +143,7 @@ VALUE resp_get_string(struct um_stream *stream, ulong len, VALUE out_buffer) {
 
   if (NIL_P(out_buffer)) return rb_utf8_str_new(start, len);
 
-  str_expand(out_buffer, len + 1);
-  char *dest_ptr = RSTRING_PTR(out_buffer);
-  memcpy(dest_ptr, start, len);
-  dest_ptr[len] = 0; // add null at end
-  rb_str_set_len(out_buffer, len);
+  str_copy_bytes(out_buffer, start, len);
   return out_buffer;
 }
 

data/ext/um/um_stream_class.c

@@ -52,36 +52,18 @@ VALUE Stream_initialize(VALUE self, VALUE machine, VALUE fd) {
   return self;
 }
 
-VALUE Stream_get_line(VALUE self) {
+VALUE Stream_get_line(VALUE self, VALUE buf, VALUE limit) {
   struct um_stream *stream = RTYPEDDATA_DATA(self);
   if (unlikely(stream->eof)) return Qnil;
 
-  return stream_get_line(stream);
+  return stream_get_line(stream, buf, NUM2LONG(limit));
 }
 
-VALUE Stream_get_string(VALUE self, VALUE len) {
+VALUE Stream_get_string(VALUE self, VALUE buf, VALUE len) {
   struct um_stream *stream = RTYPEDDATA_DATA(self);
   if (unlikely(stream->eof)) return Qnil;
 
-  return stream_get_string(stream, NUM2ULONG(len));
-}
-
-VALUE Stream_resp_get_line(VALUE self) {
-  struct um_stream *stream = RTYPEDDATA_DATA(self);
-  if (unlikely(stream->eof)) return Qnil;
-
-  VALUE line = resp_get_line(stream, Qnil);
-  RB_GC_GUARD(line);
-  return line;
-}
-
-VALUE Stream_resp_get_string(VALUE self, VALUE len) {
-  struct um_stream *stream = RTYPEDDATA_DATA(self);
-  if (unlikely(stream->eof)) return Qnil;
-
-  VALUE str = resp_get_string(stream, NUM2ULONG(len), Qnil);
-  RB_GC_GUARD(str);
-  return str;
+  return stream_get_string(stream, buf, NUM2LONG(len));
 }
 
 VALUE Stream_resp_decode(VALUE self) {
@@ -109,11 +91,8 @@ void Init_Stream(void) {
 
   rb_define_method(cStream, "initialize", Stream_initialize, 2);
   
-  rb_define_method(cStream, "get_line", Stream_get_line, 0);
-  rb_define_method(cStream, "get_string", Stream_get_string, 1);
-
-  rb_define_method(cStream, "resp_get_line", Stream_resp_get_line, 0);
-  rb_define_method(cStream, "resp_get_string", Stream_resp_get_string, 1);
+  rb_define_method(cStream, "get_line", Stream_get_line, 2);
+  rb_define_method(cStream, "get_string", Stream_get_string, 2);
 
   rb_define_method(cStream, "resp_decode", Stream_resp_decode, 0);
   

data/lib/uringmachine/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 class UringMachine
-  VERSION = '0.11.1'
+  VERSION = '0.12'
 end

data/test/test_stream.rb

@@ -15,41 +15,92 @@ class StreamTest < StreamBaseTest
     machine.write(@wfd, "foo\nbar\r\nbaz")
     machine.close(@wfd)
 
-    assert_equal 'foo', @stream.get_line
-    assert_equal 'bar', @stream.get_line
-    assert_nil @stream.get_line
+    assert_equal 'foo', @stream.get_line(nil, 0)
+    assert_equal 'bar', @stream.get_line(nil, 0)
+    assert_nil @stream.get_line(nil, 0)
+  end
+
+  def test_get_line_with_buf
+    machine.write(@wfd, "foo\nbar\r\nbaz")
+    machine.close(@wfd)
+
+    buf = +''
+    ret = @stream.get_line(buf, 0)
+    assert_equal 'foo', buf
+    assert_equal ret, buf
+
+    ret = @stream.get_line(buf, 0)
+    assert_equal 'bar', buf
+    assert_equal ret, buf
+  end
+
+  def test_get_line_with_positive_maxlen
+    machine.write(@wfd, "foobar\r\n")
+    machine.close(@wfd)
+
+    buf = +''
+    ret = @stream.get_line(buf, 3)
+    assert_nil ret
+    assert_equal '', buf
+
+    # verify that stream pos has not changed
+    ret = @stream.get_line(buf, 0)
+    assert_equal 'foobar', buf
+    assert_equal ret, buf
+  end
+
+  def test_get_line_with_negative_maxlen
+    machine.write(@wfd, "foobar\r\n")
+    machine.close(@wfd)
+
+    buf = +''
+    ret = @stream.get_line(buf, -3)
+    assert_nil ret
+    assert_equal '', buf
+
+    # verify that stream pos has not changed
+    ret = @stream.get_line(buf, 0)
+    assert_equal 'foobar', buf
+    assert_equal ret, buf
   end
 
   def test_get_string
     machine.write(@wfd, "foobarbazblahzzz")
     machine.close(@wfd)
 
-    assert_equal 'foobar', @stream.get_string(6)
-    assert_equal 'baz', @stream.get_string(3)
-    assert_equal 'blah', @stream.get_string(4)
-    assert_nil @stream.get_string(4)
+    assert_equal 'foobar', @stream.get_string(nil, 6)
+    assert_equal 'baz', @stream.get_string(nil, 3)
+    assert_equal 'blah', @stream.get_string(nil, 4)
+    assert_nil @stream.get_string(nil, 4)
   end
-end
 
-class StreamRespTest < StreamBaseTest
-  def test_trdp_get_line
-    machine.write(@wfd, "foo\r\nbarbar\r\nbaz\n")
+  def test_get_string_with_buf
+    machine.write(@wfd, "foobarbazblahzzz")
     machine.close(@wfd)
-    
-    assert_equal 'foo', @stream.resp_get_line
-    assert_equal 'barbar', @stream.resp_get_line
-    assert_nil @stream.resp_get_line
+
+    buf = +''
+    ret = @stream.get_string(buf, 6)
+    assert_equal 'foobar', buf
+    assert_equal ret, buf
+
+    ret = @stream.get_string(buf, 3)
+    assert_equal 'baz', buf
+    assert_equal ret, buf
   end
 
-  def test_resp_get_string
-    machine.write(@wfd, "foo\r\nbarbar\r\nbaz\n")
+  def test_get_string_with_negative_len
+    machine.write(@wfd, "foobar")
     machine.close(@wfd)
 
-    assert_equal 'foo', @stream.resp_get_string(3)
-    assert_equal 'barbar', @stream.resp_get_string(6)
-    assert_nil @stream.resp_get_string(3)
+    ret = @stream.get_string(nil, -12)
+    assert_equal 'foobar', ret
+
+    ret = @stream.get_string(nil, -4)
+    assert_nil ret
   end
+end
 
+class StreamRespTest < StreamBaseTest
   def test_resp_decode
     machine.write(@wfd, "+foo bar\r\n")
     assert_equal "foo bar", @stream.resp_decode

data/test/test_um.rb

@@ -271,7 +271,7 @@ class PeriodicallyTest < UMBaseTest
     rescue Cancel
       cancel = 1
     end
-    machine.snooze
+    2.times { machine.snooze }
     assert_equal 0, machine.pending_count
     t1 = monotonic_clock
     assert_in_range 0.05..0.08, t1 - t0

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: uringmachine
 version: !ruby/object:Gem::Version
-  version: 0.11.1
+  version: '0.12'
 platform: ruby
 authors:
 - Sharon Rosner