uringmachine 0.24.0 → 0.25.0

data/lib/uringmachine.rb

@@ -6,27 +6,52 @@ require 'uringmachine/dns_resolver'
 
 UM = UringMachine
 
+# A UringMachine instance provides an interface for performing I/O operations
+# and automatically switching between fibers. A single UringMachine instance
+# should be used for each thread.
 class UringMachine
-  def fiber_map
-    @fiber_map ||= {}
+
+  # Returns the set of running fibers.
+  #
+  # return [Set]
+  def fiber_set
+    @fiber_set ||= Set.new
   end
 
+  # Terminate is an exception used to terminate fibers without further bubbling.
   class Terminate < Exception
   end
 
+  # Creates a new fiber and schedules it to be ran.
+  #
+  # @param value [any] Value to be passed to the given block
+  # @param klass [Class] fiber class
+  # @param block [Proc] block to run in fiber
+  # @return [Fiber] new fiber
   def spin(value = nil, klass = Fiber, &block)
-    fiber = klass.new { |v| run_block_in_fiber(block, fiber, v) }
+    fiber = klass.new(blocking: false) { |v| run_block_in_fiber(block, fiber, v) }
     self.schedule(fiber, value)
 
-    fiber_map[fiber] = fiber
+    fiber_set << fiber
+    fiber
   end
 
+  # Runs the given block in the given fiber. This method is used to run fibers
+  # indirectly.
+  #
+  # @param fiber [Fiber] fiber
+  # @param block [Proc] block to run
+  # @return [Fiber]
   def run(fiber, &block)
     run_block_in_fiber(block, fiber, nil)
     self.schedule(fiber, nil)
-    fiber_map[fiber] = fiber
+    fiber_set << fiber
+    fiber
   end
 
+  # Waits for the given fibers to terminate.
+  #
+  # @return [Array<any>] return values of the given fibers
   def join(*fibers)
     results = fibers.inject({}) { |h, f| h[f] = nil; h }
     queue = nil
@@ -51,6 +76,11 @@ class UringMachine
     fibers.size == 1 ? values.first : values
   end
 
+  # Waits for the given fibers to terminate, without collecting their return
+  # values.
+  #
+  # @param fibers [Fiber, Array<Fiber>] fibers to wait for
+  # @return [Integer] number of fibers
   def await_fibers(fibers)
     if fibers.is_a?(Fiber)
       f = fibers
@@ -80,13 +110,50 @@ class UringMachine
     fibers.count
   end
 
+  # Resolves a hostname to an IP address by performing a DNS query.
+  #
+  # @param hostname [String] hostname
+  # @param type [Symbol] record type
+  # @return [String] IP address
   def resolve(hostname, type = :A)
     @resolver ||= DNSResolver.new(self)
     @resolver.resolve(hostname, type)
   end
 
+  # Watches for filesystem events using inotify in an infinite loop, yielding
+  # incoming events to the given block.
+  #
+  # @param root [String] directory to watch
+  # @param mask [Integer] event mask
+  # @return [void]
+  def file_watch(root, mask)
+    fd = UM.inotify_init
+    wd_map = {}
+    recursive_file_watch(fd, root, wd_map, mask)
+    while true
+      events = inotify_get_events(fd)
+      events.each do |event|
+        if event[:mask] | UM::IN_IGNORED == UM::IN_IGNORED
+          wd_map.delete(event[:wd])
+          next
+        end
+        transformed_event = transform_file_watch_event(event, wd_map)
+        if event[:mask] == UM::IN_CREATE | UM::IN_ISDIR
+          recursive_file_watch(fd, transformed_event[:fn], wd_map, mask)
+        end
+        yield transformed_event
+      end
+    end
+  ensure
+    close_async(fd)
+  end
+
   private
 
+  # @param block [Proc]
+  # @param fiber [Fiber]
+  # @param value [any]
+  # @return [void]
   def run_block_in_fiber(block, fiber, value)
     ret = block.(value)
     fiber.set_result(ret)
@@ -95,13 +162,15 @@ class UringMachine
   ensure
     fiber.mark_as_done
     # cleanup
-    fiber_map.delete(fiber)
+    fiber_set.delete(fiber)
     self.notify_done_listeners(fiber)
 
     # switch away to a different fiber
     self.switch
   end
 
+  # @param fiber [Fiber]
+  # @return [void]
   def notify_done_listeners(fiber)
     listeners = fiber.done_listeners
     return if !listeners
@@ -109,25 +178,53 @@ class UringMachine
     listeners.each { self.push(it, fiber) }
   end
 
+  # @param fd [Integer] inotify fd
+  # @param dir [String] directory path
+  # @param wd_map [Hash] hash mapping wd to directory
+  # @param mask [Integer] inotify event mask
+  # @return [void]
+  def recursive_file_watch(fd, dir, wd_map, mask)
+    wd = UM.inotify_add_watch(fd, dir, mask)
+    wd_map[wd] = dir
+    Dir[File.join(dir, '*')].each do
+      recursive_file_watch(fd, it, wd_map, mask) if File.directory?(it)
+    end
+  end
+
+  # @param event [Hash] inotify event
+  # @param wd_map [Hash] hash mapping wd to directory
+  def transform_file_watch_event(event, wd_map)
+    {
+      mask: event[:mask],
+      fn: File.join(wd_map[event[:wd]], event[:name])
+    }
+  end
+
+  # Fiber extensions
   module FiberExtensions
     attr_reader :result, :done, :done_listeners
 
+    # Marks the fiber as done (terminated)
     def mark_as_done
       @done = true
     end
 
+    # Sets the fiber return value
     def set_result(value)
       @result = value
     end
 
+    # Returns true if the fiber is done (terminated)
     def done?
       @done
     end
 
+    # Adds the given queue to the list of done listeners
     def add_done_listener(queue)
       (@done_listeners ||= []) << queue
     end
 
+    # Returns the fiber's associated mailbox
     def mailbox
       @mailbox ||= UM::Queue.new
     end
@@ -137,7 +234,10 @@ class UringMachine
     include FiberExtensions
   end
 
+  # Thread extensions
   module ThreadExtensions
+
+    # Returns the thread's associated UringMachine instance
     def machine
       @machine ||= UM.new
     end

data/test/helper.rb

@@ -85,4 +85,19 @@ class UMBaseTest < Minitest::Test
       @@port += 1
     end
   end
+
+  def make_tmp_file_tree(dir, spec)
+    FileUtils.mkdir(dir) rescue nil
+    spec.each do |k, v|
+      fn = File.join(dir, k.to_s)
+      case v
+      when String
+        IO.write(fn, v)
+      when Hash
+        FileUtils.mkdir(fn) rescue nil
+        make_tmp_file_tree(fn, v)
+      end
+    end
+    dir
+  end
 end

data/test/test_fiber_scheduler.rb

@@ -112,6 +112,43 @@ class FiberSchedulerTest < UMBaseTest
     o.close rescue nil
   end
 
+  def test_fiber_scheduler_io_with_machine_spin
+    i, o = IO.pipe
+    buffer = []
+
+    f1 = machine.spin do
+      sleep 0.01
+      o.write 'foo'
+      buffer << :f1
+    end
+
+    f2 = machine.spin do
+      sleep 0.02
+      o.write 'bar'
+      buffer << :f2
+      o.close
+    end
+
+    f3 = machine.spin do
+      str = i.read
+      buffer << str
+    end
+
+    machine.join(f1, f2, f3)
+    assert_equal [true] * 3, [f1, f2, f3].map(&:done?)
+    assert_equal [:f1, :f2, 'foobar'], buffer
+
+    assert_equal({
+      kernel_sleep: 2,
+      io_write: 2,
+      io_read: 3,
+      io_close: 1
+    }, scheduler_calls_tally)
+  ensure
+    i.close rescue nil
+    o.close rescue nil
+  end
+
   def test_io_read_with_timeout
     i, o = IO.pipe
     i.timeout = 0.01

data/test/test_ssl.rb

@@ -0,0 +1,85 @@
+# frozen_string_literal: true
+
+require_relative 'helper'
+require 'openssl'
+require 'localhost/authority'
+
+class SSLTest < UMBaseTest
+  def setup
+    super
+    authority = Localhost::Authority.fetch
+    @server_ctx = authority.server_context
+  end
+
+  def test_ssl_stock
+    sock1, sock2 = UNIXSocket.pair
+
+    s1 = OpenSSL::SSL::SSLSocket.new(sock1, @server_ctx)
+    s1.sync_close = true
+    s2 = OpenSSL::SSL::SSLSocket.new(sock2, OpenSSL::SSL::SSLContext.new)
+    s2.sync_close = true
+
+    t = Thread.new { s1.accept rescue nil }
+
+    s2.connect
+
+    assert_equal 6, s1.write('foobar')
+    assert_equal 'foobar', s2.readpartial(6)
+  ensure
+    t.join(0.1)
+    sock1&.close rescue nil
+    sock2&.close rescue nil
+  end
+
+  def test_ssl_custom_bio
+    sock1, sock2 = UNIXSocket.pair
+
+    s1 = OpenSSL::SSL::SSLSocket.new(sock1, @server_ctx)
+    s1.sync_close = true
+    s2 = OpenSSL::SSL::SSLSocket.new(sock2, OpenSSL::SSL::SSLContext.new)
+    s2.sync_close = true
+
+    @machine.ssl_set_bio(s2)
+    assert_equal true, s2.instance_variable_get(:@__um_bio__)
+
+    t = Thread.new { s1.accept rescue nil }
+
+    assert_equal 0, @machine.metrics[:total_ops]
+    s2.connect
+    refute_equal 0, @machine.metrics[:total_ops]
+
+    assert_equal 6, s1.write('foobar')
+    assert_equal 'foobar', s2.readpartial(6)
+  ensure
+    t&.join(0.1)
+    sock1&.close rescue nil
+    sock2&.close rescue nil
+  end
+
+  def test_ssl_read_write
+    sock1, sock2 = UNIXSocket.pair
+
+    s1 = OpenSSL::SSL::SSLSocket.new(sock1, @server_ctx)
+    s1.sync_close = true
+    s2 = OpenSSL::SSL::SSLSocket.new(sock2, OpenSSL::SSL::SSLContext.new)
+    s2.sync_close = true
+
+    @machine.ssl_set_bio(s2)
+    assert_equal true, s2.instance_variable_get(:@__um_bio__)
+
+    t = Thread.new { s1.accept rescue nil }
+
+    assert_equal 0, @machine.metrics[:total_ops]
+    s2.connect
+    refute_equal 0, @machine.metrics[:total_ops]
+
+    assert_equal 6, @machine.ssl_write(s1, 'foobar', 6)
+    buf = +''
+    assert_equal 6, @machine.ssl_read(s2, buf, 6)
+    assert_equal 'foobar', buf
+  ensure
+    t&.join(0.1)
+    sock1&.close rescue nil
+    sock2&.close rescue nil
+  end
+end

data/test/test_stream.rb

@@ -163,6 +163,8 @@ class StreamRespTest < StreamBaseTest
     machine.write(@wfd, "%2\r\n+a\r\n:42\r\n+b\r\n*3\r\n+foo\r\n+bar\r\n+baz\r\n")
     assert_equal({ 'a' => 42, 'b' => ['foo', 'bar', 'baz'] }, @stream.resp_decode)
 
+    machine.write(@wfd, "%2\r\n+a\r\n:42\r\n+b\r\n*3\r\n+foo\r\n+xbar\r\n+yybaz\r\n")
+    assert_equal({ 'a' => 42, 'b' => ['foo', 'xbar', 'yybaz'] }, @stream.resp_decode)
   end
 
   def test_resp_encode
@@ -181,6 +183,9 @@ class StreamRespTest < StreamBaseTest
     assert_equal "*2\r\n$3\r\nfoo\r\n$3\r\nbar\r\n",
       s.resp_encode(+'', ['foo', 'bar'])
 
+    assert_equal "*2\r\n$3\r\nfoo\r\n$4\r\nxbar\r\n",
+      s.resp_encode(+'', ['foo', 'xbar'])
+
     assert_equal "%2\r\n$3\r\nfoo\r\n$3\r\nbar\r\n$3\r\nbaz\r\n:42\r\n",
       s.resp_encode(+'', { 'foo' => 'bar', 'baz' => 42 })
   end
@@ -191,10 +196,16 @@ class StreamRespTest < StreamBaseTest
     assert_equal "*2\r\n$3\r\nfoo\r\n$3\r\nbar\r\n",
       s.resp_encode_cmd(+'', 'foo', 'bar')
 
+    assert_equal "*2\r\n$3\r\nfoo\r\n$4\r\nxbar\r\n",
+      s.resp_encode_cmd(+'', 'foo', 'xbar')
+
     assert_equal "*2\r\n$3\r\nfoo\r\n$3\r\nbar\r\n",
       s.resp_encode_cmd(+'', 'foo', :bar)
 
     assert_equal "*2\r\n$3\r\nfoo\r\n$3\r\n123\r\n",
       s.resp_encode_cmd(+'', 'foo', 123)
+
+    assert_equal "*4\r\n$3\r\nset\r\n$6\r\nfoobar\r\n$2\r\nnx\r\n$2\r\nxx\r\n",
+      s.resp_encode_cmd(+'', :set, 'foobar', :nx, :xx)
   end
 end

data/test/test_um.rb

@@ -540,7 +540,7 @@ class ReadTest < UMBaseTest
 
     machine.close(w)
 
-    res = machine.read(r, read_buffer)
+    res = machine.read(r, read_buffer, 6)
     assert_equal 3, res
     assert_equal 'bar', read_buffer.get_string(0, 3)
   end
@@ -570,7 +570,7 @@ class ReadTest < UMBaseTest
   def test_read_invalid_buffer
     r, _w = UM.pipe
     assert_raises(UM::Error) {
-      machine.read(r, [])
+      machine.read(r, [], 3)
     }
   end
 
@@ -2725,3 +2725,157 @@ class ProfileModeTest < UMBaseTest
     assert_in_range 0.01..0.015, machine.metrics[:time_total_wait]
   end
 end
+
+class FileWatchTest < UMBaseTest
+  def setup
+    super
+    @root = "/tmp/um_#{SecureRandom.hex}"
+    make_tmp_file_tree(@root, {
+      'foo.txt': 'foo',
+      'bar.txt': 'bar',
+      'foo': {
+        'baz.txt': 'baz'
+      }
+    })
+  end
+
+  def test_inotify_get_events
+    fd = UM.inotify_init
+    assert_kind_of Integer, fd
+    assert fd > 0
+
+    mask = UM::IN_CREATE | UM::IN_DELETE | UM::IN_CLOSE_WRITE
+
+    assert_raises(Errno::EBADF)   { UM.inotify_add_watch(-1, @root, mask) }
+    assert_raises(Errno::EINVAL)  { UM.inotify_add_watch(fd, @root, 0) }
+    non_existent = "/tmp/um_#{SecureRandom.hex}"
+    assert_raises(Errno::ENOENT)  { UM.inotify_add_watch(fd, non_existent, mask) }
+
+    wd = UM.inotify_add_watch(fd, @root, mask)
+    assert_equal 1, wd
+
+    fn = File.join(@root, 'foo.txt')
+    IO.write(fn, 'foofoo')
+
+    events = machine.inotify_get_events(fd)
+    assert_equal [
+      {
+        wd: wd,
+        mask: UM::IN_CLOSE_WRITE,
+        name: 'foo.txt'
+      }
+    ], events
+  end
+
+  def write_file(fn, data)
+    File.open(fn, 'w+') { it.sync = true; it.syswrite('foofoo') }
+  end
+
+  def test_file_watch
+    events = []
+    f = machine.spin do
+      mask = UM::IN_CREATE | UM::IN_DELETE | UM::IN_CLOSE_WRITE
+      machine.file_watch(@root, mask) do |evt|
+        events << evt
+      end
+    rescue => e
+      p e
+      p e.backtrace
+      exit!
+    end
+
+    3.times { machine.snooze }
+    assert_equal [], events
+
+    # modify foo.txt
+    fn = File.join(@root, 'foo.txt')
+    write_file(fn, 'foofoo')
+    machine.sleep(0.05)
+    assert_equal [
+      { mask: UM::IN_CLOSE_WRITE, fn: File.join(@root, 'foo.txt') }
+    ], events
+
+    # modify foo/baz.txt
+    fn = File.join(@root, 'foo/baz.txt')
+    write_file(fn, 'bazbaz')
+    machine.sleep(0.05)
+    assert_equal [
+      { mask: UM::IN_CLOSE_WRITE, fn: File.join(@root, 'foo.txt') },
+      { mask: UM::IN_CLOSE_WRITE, fn: File.join(@root, 'foo/baz.txt') }
+    ], events
+
+    # create bar dir
+    fn = File.join(@root, 'bar')
+    FileUtils.mkdir(fn)
+    machine.sleep(0.05)
+    assert_equal [
+      { mask: UM::IN_CLOSE_WRITE, fn: File.join(@root, 'foo.txt') },
+      { mask: UM::IN_CLOSE_WRITE, fn: File.join(@root, 'foo/baz.txt') },
+      { mask: UM::IN_CREATE | UM::IN_ISDIR, fn: File.join(@root, 'bar') }
+    ], events
+
+    # create bar/baz.txt
+    fn = File.join(@root, 'bar/baz.txt')
+    write_file(fn, 'bazbaz')
+    machine.sleep(0.05)
+    assert_equal [
+      { mask: UM::IN_CLOSE_WRITE, fn: File.join(@root, 'foo.txt') },
+      { mask: UM::IN_CLOSE_WRITE, fn: File.join(@root, 'foo/baz.txt') },
+      { mask: UM::IN_CREATE | UM::IN_ISDIR, fn: File.join(@root, 'bar') },
+      { mask: UM::IN_CREATE, fn: File.join(@root, 'bar/baz.txt') },
+      { mask: UM::IN_CLOSE_WRITE, fn: File.join(@root, 'bar/baz.txt') },
+    ], events
+
+    # delete foo/baz.txt
+    fn = File.join(@root, 'foo/baz.txt')
+    FileUtils.rm(fn)
+    machine.sleep(0.05)
+    assert_equal [
+      { mask: UM::IN_CLOSE_WRITE, fn: File.join(@root, 'foo.txt') },
+      { mask: UM::IN_CLOSE_WRITE, fn: File.join(@root, 'foo/baz.txt') },
+      { mask: UM::IN_CREATE | UM::IN_ISDIR, fn: File.join(@root, 'bar') },
+      { mask: UM::IN_CREATE, fn: File.join(@root, 'bar/baz.txt') },
+      { mask: UM::IN_CLOSE_WRITE, fn: File.join(@root, 'bar/baz.txt') },
+      { mask: UM::IN_DELETE, fn: File.join(@root, 'foo/baz.txt') }
+    ], events
+
+    # delete bar
+    fn = File.join(@root, 'bar')
+    FileUtils.rm_rf(fn)
+    machine.sleep(0.05)
+    assert_equal [
+      { mask: UM::IN_CLOSE_WRITE, fn: File.join(@root, 'foo.txt') },
+      { mask: UM::IN_CLOSE_WRITE, fn: File.join(@root, 'foo/baz.txt') },
+      { mask: UM::IN_CREATE | UM::IN_ISDIR, fn: File.join(@root, 'bar') },
+      { mask: UM::IN_CREATE, fn: File.join(@root, 'bar/baz.txt') },
+      { mask: UM::IN_CLOSE_WRITE, fn: File.join(@root, 'bar/baz.txt') },
+      { mask: UM::IN_DELETE, fn: File.join(@root, 'foo/baz.txt') },
+      { mask: UM::IN_DELETE, fn: File.join(@root, 'bar/baz.txt') },
+      { mask: UM::IN_DELETE | UM::IN_ISDIR, fn: File.join(@root, 'bar') }
+    ], events
+
+    # recreate bar, bar/baz.txt
+    fn = File.join(@root, 'bar')
+    FileUtils.mkdir(fn)
+    machine.sleep(0.05)
+    fn = File.join(@root, 'bar/baz.txt')
+    write_file(fn, 'bazbaz')
+    machine.sleep(0.05)
+    assert_equal [
+      { mask: UM::IN_CLOSE_WRITE, fn: File.join(@root, 'foo.txt') },
+      { mask: UM::IN_CLOSE_WRITE, fn: File.join(@root, 'foo/baz.txt') },
+      { mask: UM::IN_CREATE | UM::IN_ISDIR, fn: File.join(@root, 'bar') },
+      { mask: UM::IN_CREATE, fn: File.join(@root, 'bar/baz.txt') },
+      { mask: UM::IN_CLOSE_WRITE, fn: File.join(@root, 'bar/baz.txt') },
+      { mask: UM::IN_DELETE, fn: File.join(@root, 'foo/baz.txt') },
+      { mask: UM::IN_DELETE, fn: File.join(@root, 'bar/baz.txt') },
+      { mask: UM::IN_DELETE | UM::IN_ISDIR, fn: File.join(@root, 'bar') },
+      { mask: UM::IN_CREATE | UM::IN_ISDIR, fn: File.join(@root, 'bar') },
+      { mask: UM::IN_CREATE, fn: File.join(@root, 'bar/baz.txt') },
+      { mask: UM::IN_CLOSE_WRITE, fn: File.join(@root, 'bar/baz.txt') },
+    ], events
+  ensure
+    machine.schedule(f, UM::Terminate.new)
+    machine.join(f)
+  end
+end

data/uringmachine.gemspec

@@ -15,14 +15,8 @@ Gem::Specification.new do |s|
     "changelog_uri" => "https://github.com/digital-fabric/uringmachine/blob/master/CHANGELOG.md"
   }
   s.rdoc_options = ["--title", "UringMachine", "--main", "README.md"]
-  s.extra_rdoc_files = ["README.md"]
+  s.extra_rdoc_files = `git ls-files -z *.rdoc *.md lib/*.rb lib/**/*.rb ext/um/*.c ext/um/*.h benchmark/*.md benchmark/*.png`.split("\x0")
   s.extensions = ["ext/um/extconf.rb"]
   s.require_paths = ["lib"]
   s.required_ruby_version = '>= 3.5'
-
-  s.add_development_dependency  'rake-compiler',        '~>1.3.0'
-  s.add_development_dependency  'minitest',             '~>5.26.2'
-  s.add_development_dependency  'benchmark'
-  s.add_development_dependency  'benchmark-ips'
-  s.add_development_dependency  'http_parser.rb',       '~>0.8.0'
 end