uringmachine 0.11 → 0.12

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 8297cccb35ae1c8086889c2400db8bb884b0dfccdbef561caa0a4e9f80b21053
-  data.tar.gz: 990e84bb2f975de00739150cff966bbf97a551ba135714eb92bc3b86cd14b5d0
+  metadata.gz: 1f8224da7817436bd74a52e650362c9b77d5dbc8877c0fa1cefd1c52e57ceb03
+  data.tar.gz: c7bb88c42bf59a9d1f105e2356c4f71f0fec49c7cbbe662744465ee3b34d5141
 SHA512:
-  metadata.gz: f66149736c1cf58f8df65d387b81ff070fd890e99aa7c8aa806a5afcb04b8248b78c95c1673d30b7d3270c81e05bc0b84afdb025d2a0a2f02beca5977196b321
-  data.tar.gz: 7b6e6c31193da3a1e3e9a6fa627c84ae0f4d35c36f5fa636595acf1a013b71cd419073bb78be04130333796bed05e0ce9d1bfa74ab195608346cd689ecf9abaf
+  metadata.gz: 39d930e9f4d8a17676c28e4bf24f38b585dbd0cd124316272778abb285391be4bb9ecf14452d19568ffa8f6a0144c6a98b2e3ee1f50d58ec7c5101333f5de5e7
+  data.tar.gz: 95468f71d402a2a8abf675dbbae6e94b4e1919c5552f0b831b7387016d3fb24271756585cf66283130ed3440818ccc94f504d73a46d2ab8f9faa4607b0cc822b

data/CHANGELOG.md

@@ -1,3 +1,13 @@
+# 2025-06-03 Version 0.12
+
+- Add buffer, maxlen params to `Stream#get_line`
+- Add buffer param to `Stream#get_string`
+- Remove `Stream#resp_get_line`, `Stream#resp_get_string` methods
+
+# 2025-06-02 Version 0.11.1
+
+- Fix `UM::Stream` behaviour on GC
+
 # 2025-06-02 Version 0.11
 
 - Implement `UM::Stream` class for read streams

data/TODO.md

@@ -22,3 +22,27 @@
 When doing a `call`, we need to provide a mailbox for the response. can this be
 automatic?
 
+# streams
+
+We're still missing:
+
+- limit on line length in `get_line`
+- ability to supply buffer to `get_line` and `get_string`
+- allow read to eof, maybe with `read_to_eof`
+
+For the sake of performance, simplicity and explicitness, we change the API as follows:
+
+```ruby
+stream.get_line(buf, limit)
+# the defaults:
+stream.get_line(nil, -1)
+
+stream.get_string(len, buf)
+# defaults:
+stream.get_string(len, nil)
+
+# and
+stream.read_to_eof(buf)
+# defaults:
+stream.read_to_eof(nil)
+```

data/examples/bm_http_parse.rb

@@ -1,16 +1,17 @@
 # frozen_string_literal: true
 
-require 'bundler/inline'
+# require 'bundler/inline'
 
-gemfile do
-  source 'https://rubygems.org'
-  gem 'uringmachine', path: '..'
-  gem 'benchmark-ips'
-  gem 'http_parser.rb'
-end
+# gemfile do
+#   source 'https://rubygems.org'
+#   gem 'uringmachine', path: '..'
+#   gem 'benchmark-ips'
+#   gem 'http_parser.rb'
+# end
 
-require 'benchmark/ips'
+require 'bundler/setup'
 require 'uringmachine'
+require 'benchmark/ips'
 require 'http/parser'
 
 $machine = UM.new
@@ -19,6 +20,8 @@ HTTP_MSG = "GET /foo/bar HTTP/1.1\r\nServer: foobar.com\r\nFoo: bar\r\n\r\n"
 
 $count = 0
 
+STDOUT.sync = true
+
 def parse_http_parser
   current_fiber = Fiber.current
   $count += 1
@@ -45,10 +48,16 @@ def parse_http_parser
   end
 
   $machine.write(w.fileno, HTTP_MSG)
-  $machine.yield
+  ret = $machine.yield
+  ret
+rescue Exception => e
+  p e: e
+  exit
 ensure
-  $machine.close(r.fileno)
-  $machine.close(w.fileno)
+  r.close rescue nil
+  w.close rescue nil
+  # $machine.close(r.fileno) rescue nil
+  # $machine.close(w.fileno) rescue nil
 end
 
 require 'stringio'
@@ -100,50 +109,116 @@ def parse_headers(fd)
 end
 
 def parse_http_stringio
-  current_fiber = Fiber.current
-  r, w = IO.pipe
+  rfd, wfd = UM.pipe
+  queue = UM::Queue.new
 
   $machine.spin do
-    headers = parse_headers(r.fileno)
-    $machine.schedule(current_fiber, headers)
+    headers = parse_headers(rfd)
+    $machine.push(queue, headers)
   rescue Exception => e
     p e
     puts e.backtrace.join("\n")
     exit!
   end
 
-  $machine.write(w.fileno, HTTP_MSG)
-  $machine.yield
+  $machine.write(wfd, HTTP_MSG)
+  $machine.close(wfd)
+  $machine.shift(queue)
 ensure
-  $machine.close(r.fileno)
-  $machine.close(w.fileno)
+  ($machine.close(rfd) rescue nil) if rfd
+  ($machine.close(wfd) rescue nil) if wfd
+end
+
+def stream_parse_headers(fd)
+  stream = UM::Stream.new($machine, fd)
+
+  buf = String.new(capacity: 65536)
+  headers = stream_get_request_line(stream, buf)
+  return nil if !headers
+
+  while true
+    line = stream.get_line(buf, 0)
+    break if line.empty?
+
+    m = line.match(RE_HEADER_LINE)
+    raise "Invalid header" if !m
+
+    headers[m[1]] = m[2]
+  end
+
+  headers
 end
 
-# p parse_http_parser
-# p parse_http_stringio
-# exit
+def stream_get_request_line(stream, buf)
+  line = stream.get_line(buf, 0)
 
-GC.disable
+  m = line.match(RE_REQUEST_LINE)
+  return nil if !m
+
+  {
+    'method'   => m[1].downcase,
+    'path'     => m[2],
+    'protocol' => m[3].downcase
+  }
+end
+
+def parse_http_stream
+  rfd, wfd = UM.pipe
+  queue = UM::Queue.new
+
+  $machine.spin do
+    headers = stream_parse_headers(rfd)
+    $machine.push(queue, headers)
+  rescue Exception => e
+    p e
+    puts e.backtrace.join("\n")
+    exit!
+  end
+
+  $machine.write(wfd, HTTP_MSG)
+  $machine.shift(queue)
+ensure
+  ($machine.close(rfd) rescue nil) if rfd
+  ($machine.close(wfd) rescue nil) if wfd
+end
+
+def compare_allocs
+  GC.disable
+  x = 1000
+  p(
+    alloc_http_parser:  alloc_count { x.times { parse_http_parser } },
+    alloc_stringio:     alloc_count { x.times { parse_http_stringio } },
+    alloc_stream:       alloc_count { x.times { parse_http_stream } }
+  )
+ensure
+  GC.enable
+end
+
+def object_count
+  counts = ObjectSpace.count_objects
+  counts[:TOTAL] - counts[:FREE]
+end
 
 def alloc_count
-  count0 = ObjectSpace.count_objects[:TOTAL]
+  GC.start
+  count0 = object_count
   yield
-  count1 = ObjectSpace.count_objects[:TOTAL]
+  # GC.start
+  count1 = object_count
   count1 - count0
 end
 
-X = 100
-p(
-  alloc_http_parser: alloc_count { X.times { parse_http_parser } },
-  alloc_stringio: alloc_count { X.times { parse_http_stringio } }
-)
-exit
+def benchmark
+  Benchmark.ips do |x|
+    x.config(:time => 5, :warmup => 3)
 
-Benchmark.ips do |x|
-  x.config(:time => 5, :warmup => 3)
+    x.report("http_parser") { parse_http_parser }
+    x.report("stringio") { parse_http_stringio }
+    x.report("stream") { parse_http_stream }
 
-  x.report("http_parser") { parse_http_parser }
-  x.report("homegrown") { parse_http_stringio }
-
-  x.compare!
+    x.compare!
+  end
 end
+
+compare_allocs
+benchmark

data/ext/um/um.h

@@ -270,10 +270,8 @@ VALUE um_queue_pop(struct um *machine, struct um_queue *queue);
 VALUE um_queue_unshift(struct um *machine, struct um_queue *queue, VALUE value);
 VALUE um_queue_shift(struct um *machine, struct um_queue *queue);
 
-int stream_read_more(struct um_stream *stream);
-
-VALUE resp_get_line(struct um_stream *stream, VALUE out_buffer);
-VALUE resp_get_string(struct um_stream *stream, ulong len, VALUE out_buffer);
+VALUE stream_get_line(struct um_stream *stream, VALUE buf, ssize_t maxlen);
+VALUE stream_get_string(struct um_stream *stream, VALUE buf, ssize_t len);
 VALUE resp_decode(struct um_stream *stream, VALUE out_buffer);
 void resp_encode(struct um_write_buffer *buf, VALUE obj);
 

data/ext/um/um_stream.c

@@ -1,4 +1,5 @@
 #include "um.h"
+#include <stdlib.h>
 
 static inline void stream_check_truncate_buffer(struct um_stream *stream) {
   if ((stream->pos == stream->len) && (stream->len >= 1 << 12)) {
@@ -40,10 +41,67 @@ int stream_read_more(struct um_stream *stream) {
   return 1;
 }
 
-// ensure string can hold at least len bytes
+// ensures given string can hold at least given len bytes (+trailing null)
 static inline void str_expand(VALUE str, size_t len) {
-  size_t capa = rb_str_capacity(str);
-  if (capa < len + 1) rb_str_modify_expand(str, len + 1 - capa);
+  rb_str_resize(str, len);
+}
+
+static inline void str_copy_bytes(VALUE dest, const char *src, ssize_t len) {
+  str_expand(dest, len + 1);
+  char *dest_ptr = RSTRING_PTR(dest);
+  memcpy(dest_ptr, src, len);
+  dest_ptr[len] = 0;
+  rb_str_set_len(dest, len);
+}
+
+VALUE stream_get_line(struct um_stream *stream, VALUE buf, ssize_t maxlen) {
+  char *start = RSTRING_PTR(stream->buffer) + stream->pos;
+  while (true) {
+    ssize_t pending_len = stream->len - stream->pos;
+    ssize_t search_len = pending_len;
+    ssize_t absmax_len = labs(maxlen);
+    int should_limit_len = (absmax_len > 0) && (search_len > maxlen);
+    if (should_limit_len) search_len = absmax_len;
+
+    char * lf_ptr = memchr(start, '\n', search_len);
+    if (lf_ptr) {
+      ssize_t len = lf_ptr - start;
+      if (len && (start[len - 1] == '\r')) len -= 1;
+
+      stream->pos += lf_ptr - start + 1;
+      if (NIL_P(buf)) return rb_utf8_str_new(start, len);
+
+      str_copy_bytes(buf, start, len);
+      return buf;
+    }
+    else if (should_limit_len && pending_len > search_len)
+      // maxlen
+      return Qnil;
+
+    if (!stream_read_more(stream))
+      return Qnil;
+    else
+      // update start ptr (it might have changed after reading)
+      start = RSTRING_PTR(stream->buffer) + stream->pos;
+  }
+}
+
+VALUE stream_get_string(struct um_stream *stream, VALUE buf, ssize_t len) {
+  size_t abslen = labs(len);
+  while (stream->len - stream->pos < abslen)
+    if (!stream_read_more(stream)) {
+      if (len > 0) return Qnil;
+
+      abslen = stream->len - stream->pos;
+    }
+  
+  char *start = RSTRING_PTR(stream->buffer) + stream->pos;
+  stream->pos += abslen;
+
+  if (NIL_P(buf)) return rb_utf8_str_new(start, abslen);
+
+  str_copy_bytes(buf, start, len);
+  return buf;
 }
 
 VALUE resp_get_line(struct um_stream *stream, VALUE out_buffer) {
@@ -58,14 +116,11 @@ VALUE resp_get_line(struct um_stream *stream, VALUE out_buffer) {
         VALUE str = rb_str_new(start, len + 1);
         rb_str_set_len(str, len);
         RSTRING_PTR(str)[len] = 0;
+        RB_GC_GUARD(str);
         return str;
       }
 
-      str_expand(out_buffer, len + 1);
-      char *dest_ptr = RSTRING_PTR(out_buffer);
-      memcpy(dest_ptr, start, len);
-      dest_ptr[len] = 0; // add null at end
-      rb_str_set_len(out_buffer, len);
+      str_copy_bytes(out_buffer, start, len);
       return out_buffer;
     }
 
@@ -88,11 +143,7 @@ VALUE resp_get_string(struct um_stream *stream, ulong len, VALUE out_buffer) {
 
   if (NIL_P(out_buffer)) return rb_utf8_str_new(start, len);
 
-  str_expand(out_buffer, len + 1);
-  char *dest_ptr = RSTRING_PTR(out_buffer);
-  memcpy(dest_ptr, start, len);
-  dest_ptr[len] = 0; // add null at end
-  rb_str_set_len(out_buffer, len);
+  str_copy_bytes(out_buffer, start, len);
   return out_buffer;
 }
 

data/ext/um/um_stream_class.c

@@ -37,6 +37,8 @@ static VALUE Stream_allocate(VALUE klass) {
 VALUE Stream_initialize(VALUE self, VALUE machine, VALUE fd) {
   struct um_stream *stream = RTYPEDDATA_DATA(self);
 
+  stream->self = self;
+  
   stream->machine = RTYPEDDATA_DATA(machine);
   stream->fd = NUM2ULONG(fd);
   stream->buffer = rb_utf8_str_new_literal("");
@@ -50,57 +52,18 @@ VALUE Stream_initialize(VALUE self, VALUE machine, VALUE fd) {
   return self;
 }
 
-VALUE Stream_get_line(VALUE self) {
-  struct um_stream *stream = RTYPEDDATA_DATA(self);
-  if (unlikely(stream->eof)) return Qnil;
-
-  char *start = RSTRING_PTR(stream->buffer) + stream->pos;
-  while (true) {
-    char * lf_ptr = memchr(start, '\n', stream->len - stream->pos);
-    if (lf_ptr) {
-      ulong len = lf_ptr - start;
-      if (len && (start[len - 1] == '\r')) len -= 1;
-
-      VALUE str = rb_str_new(start, len);
-      stream->pos += lf_ptr - start + 1;
-      return str;
-    }
-
-    if (!stream_read_more(stream)) return Qnil;
-  }
-}
-
-VALUE Stream_get_string(VALUE self, VALUE len) {
-  struct um_stream *stream = RTYPEDDATA_DATA(self);
-  if (unlikely(stream->eof)) return Qnil;
-
-  ulong ulen = NUM2ULONG(len);
-
-  while (stream->len - stream->pos < ulen)
-    if (!stream_read_more(stream)) return Qnil;
-  
-  char *start = RSTRING_PTR(stream->buffer) + stream->pos;
-  VALUE str = rb_utf8_str_new(start, ulen);
-  stream->pos += ulen;
-  return str;
-}
-
-VALUE Stream_resp_get_line(VALUE self) {
+VALUE Stream_get_line(VALUE self, VALUE buf, VALUE limit) {
   struct um_stream *stream = RTYPEDDATA_DATA(self);
   if (unlikely(stream->eof)) return Qnil;
 
-  VALUE line = resp_get_line(stream, Qnil);
-  RB_GC_GUARD(line);
-  return line;
+  return stream_get_line(stream, buf, NUM2LONG(limit));
 }
 
-VALUE Stream_resp_get_string(VALUE self, VALUE len) {
+VALUE Stream_get_string(VALUE self, VALUE buf, VALUE len) {
   struct um_stream *stream = RTYPEDDATA_DATA(self);
   if (unlikely(stream->eof)) return Qnil;
 
-  VALUE str = resp_get_string(stream, NUM2ULONG(len), Qnil);
-  RB_GC_GUARD(str);
-  return str;
+  return stream_get_string(stream, buf, NUM2LONG(len));
 }
 
 VALUE Stream_resp_decode(VALUE self) {
@@ -128,11 +91,8 @@ void Init_Stream(void) {
 
   rb_define_method(cStream, "initialize", Stream_initialize, 2);
   
-  rb_define_method(cStream, "get_line", Stream_get_line, 0);
-  rb_define_method(cStream, "get_string", Stream_get_string, 1);
-
-  rb_define_method(cStream, "resp_get_line", Stream_resp_get_line, 0);
-  rb_define_method(cStream, "resp_get_string", Stream_resp_get_string, 1);
+  rb_define_method(cStream, "get_line", Stream_get_line, 2);
+  rb_define_method(cStream, "get_string", Stream_get_string, 2);
 
   rb_define_method(cStream, "resp_decode", Stream_resp_decode, 0);
   

data/lib/uringmachine/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 class UringMachine
-  VERSION = '0.11'
+  VERSION = '0.12'
 end

data/test/test_stream.rb

@@ -15,41 +15,92 @@ class StreamTest < StreamBaseTest
     machine.write(@wfd, "foo\nbar\r\nbaz")
     machine.close(@wfd)
 
-    assert_equal 'foo', @stream.get_line
-    assert_equal 'bar', @stream.get_line
-    assert_nil @stream.get_line
+    assert_equal 'foo', @stream.get_line(nil, 0)
+    assert_equal 'bar', @stream.get_line(nil, 0)
+    assert_nil @stream.get_line(nil, 0)
+  end
+
+  def test_get_line_with_buf
+    machine.write(@wfd, "foo\nbar\r\nbaz")
+    machine.close(@wfd)
+
+    buf = +''
+    ret = @stream.get_line(buf, 0)
+    assert_equal 'foo', buf
+    assert_equal ret, buf
+
+    ret = @stream.get_line(buf, 0)
+    assert_equal 'bar', buf
+    assert_equal ret, buf
+  end
+
+  def test_get_line_with_positive_maxlen
+    machine.write(@wfd, "foobar\r\n")
+    machine.close(@wfd)
+
+    buf = +''
+    ret = @stream.get_line(buf, 3)
+    assert_nil ret
+    assert_equal '', buf
+
+    # verify that stream pos has not changed
+    ret = @stream.get_line(buf, 0)
+    assert_equal 'foobar', buf
+    assert_equal ret, buf
+  end
+
+  def test_get_line_with_negative_maxlen
+    machine.write(@wfd, "foobar\r\n")
+    machine.close(@wfd)
+
+    buf = +''
+    ret = @stream.get_line(buf, -3)
+    assert_nil ret
+    assert_equal '', buf
+
+    # verify that stream pos has not changed
+    ret = @stream.get_line(buf, 0)
+    assert_equal 'foobar', buf
+    assert_equal ret, buf
   end
 
   def test_get_string
     machine.write(@wfd, "foobarbazblahzzz")
     machine.close(@wfd)
 
-    assert_equal 'foobar', @stream.get_string(6)
-    assert_equal 'baz', @stream.get_string(3)
-    assert_equal 'blah', @stream.get_string(4)
-    assert_nil @stream.get_string(4)
+    assert_equal 'foobar', @stream.get_string(nil, 6)
+    assert_equal 'baz', @stream.get_string(nil, 3)
+    assert_equal 'blah', @stream.get_string(nil, 4)
+    assert_nil @stream.get_string(nil, 4)
   end
-end
 
-class StreamRespTest < StreamBaseTest
-  def test_trdp_get_line
-    machine.write(@wfd, "foo\r\nbarbar\r\nbaz\n")
+  def test_get_string_with_buf
+    machine.write(@wfd, "foobarbazblahzzz")
     machine.close(@wfd)
-    
-    assert_equal 'foo', @stream.resp_get_line
-    assert_equal 'barbar', @stream.resp_get_line
-    assert_nil @stream.resp_get_line
+
+    buf = +''
+    ret = @stream.get_string(buf, 6)
+    assert_equal 'foobar', buf
+    assert_equal ret, buf
+
+    ret = @stream.get_string(buf, 3)
+    assert_equal 'baz', buf
+    assert_equal ret, buf
   end
 
-  def test_resp_get_string
-    machine.write(@wfd, "foo\r\nbarbar\r\nbaz\n")
+  def test_get_string_with_negative_len
+    machine.write(@wfd, "foobar")
     machine.close(@wfd)
 
-    assert_equal 'foo', @stream.resp_get_string(3)
-    assert_equal 'barbar', @stream.resp_get_string(6)
-    assert_nil @stream.resp_get_string(3)
+    ret = @stream.get_string(nil, -12)
+    assert_equal 'foobar', ret
+
+    ret = @stream.get_string(nil, -4)
+    assert_nil ret
   end
+end
 
+class StreamRespTest < StreamBaseTest
   def test_resp_decode
     machine.write(@wfd, "+foo bar\r\n")
     assert_equal "foo bar", @stream.resp_decode

data/test/test_um.rb

@@ -271,7 +271,7 @@ class PeriodicallyTest < UMBaseTest
     rescue Cancel
       cancel = 1
     end
-    machine.snooze
+    2.times { machine.snooze }
     assert_equal 0, machine.pending_count
     t1 = monotonic_clock
     assert_in_range 0.05..0.08, t1 - t0

data/uringmachine.gemspec

@@ -23,4 +23,5 @@ Gem::Specification.new do |s|
   s.add_development_dependency  'rake-compiler',        '1.2.9'
   s.add_development_dependency  'minitest',             '5.25.4'
   s.add_development_dependency  'benchmark-ips',        '2.14.0'
+  s.add_development_dependency  'http_parser.rb',       '0.8.0'
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: uringmachine
 version: !ruby/object:Gem::Version
-  version: '0.11'
+  version: '0.12'
 platform: ruby
 authors:
 - Sharon Rosner
@@ -51,6 +51,20 @@ dependencies:
     - - '='
       - !ruby/object:Gem::Version
         version: 2.14.0
+- !ruby/object:Gem::Dependency
+  name: http_parser.rb
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 0.8.0
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 0.8.0
 email: sharon@noteflakes.com
 executables: []
 extensions: