uringmachine 0.19.1 → 0.21.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: ff89f34d541e086f8016156d4b5290a6a2fdc94292f04e7a3104ef9e7cda256d
-  data.tar.gz: 01dbe57b83effbab744fbd07461dcd803246a88bbf3ce1aeaf1e4eace32fd0ad
+  metadata.gz: e14941110d09a575728da68cb038a02334d151f0d384d88ad67a34b21a49d234
+  data.tar.gz: 158299106b117a973163ae480ea4e9827a0cb88eb76b2323da04fd631e8916ca
 SHA512:
-  metadata.gz: ca0e44f1121384634b4234b2cd49643f87e8255e77eb250f64fb62be8bcdcb18e6a82480e5da003322c941c9ae2def119a048797d53bc064c4e1f77796b82c4d
-  data.tar.gz: 322090fd12498525d303b63f66ec1c559ed9933e89e74d3b01c9cc5c0f3f1dcee951c767625690c288478d5e460248418bbd693fbd7c31146c7aff76cf545a5d
+  metadata.gz: f71a5b6b6740fea281df8880511b424a0c759e876e17f79a6b661453416d13c2d9d65e0c53ddee5df066ca92472571ffae8c0e9a4ca4fd745d02705a8645965d
+  data.tar.gz: baa86b9bc0009c69bfe894aae3489d43ed28d1fca44a24a9ddc7cc1625a54de6c17f65731c1a707af561942174f9584554a183a67f8428d5855e799c274db0b6

data/.github/workflows/test.yml

@@ -11,9 +11,8 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        # macos-latest uses arm64, macos-13 uses x86
         os: [ubuntu-latest]
-        ruby: ['3.4', 'head']
+        ruby: ['head']
 
     name: ${{matrix.os}}, ${{matrix.ruby}}
 
@@ -32,5 +31,5 @@ jobs:
     - name: Compile C-extension
       run: bundle exec rake compile
     - name: Run tests
-      # run: bundle exec ruby test/test_um.rb --name test_read_each_raising_2
-      run: bundle exec rake test
+      run: bundle exec ruby test/run.rb --verbose
+      # run: bundle exec rake test

data/CHANGELOG.md

@@ -1,4 +1,35 @@
-# 0.19.1 2025-01-03
+# 0.21.0 2025-12-06
+
+- Add `UM#submit`
+- Update liburing
+- Do not release GVL in um_submit if SQ does not need entering the kernel
+- Fix compilation when rb_process_status_new is not available
+- Fix um_futex_wake_transient to submit SQE, fix futex_wait usage
+- Add debug logging for key io_uring interactions
+- Add UM#mark and DEBUG_MARK for debugging specific UM instances
+- Short-circuit zero-length writes
+- Add optional file_offset argument to #read, #write. Add optional len and file_off
+set arguments to #write_async
+- Add support for specifying SQPOLL mode and SQ idle timeout in `UM#initialize`
+- Add support for specifying number of SQ entries in `UM#initialize`
+- Implement global worker pool for blocking operations in fiber scheduler
+- Finish implementing all fiber scheduler hooks
+- Add `UM#select`
+- Add `UM#wakeup`
+- Add `UM#total_op_count`
+
+# 0.20.0 2025-11-26
+
+- Add `UM.pidfd_open`, `UM.pidfd_send_signal` methods
+- Add `#waitid`, `#waitid_status` methods, remove `#waitpid`
+- Set minimal kernel version to 6.7
+- Add `UM::Error` exception class
+- Add support for `IO::Buffer` in all I/O methods
+- Fix and improve mutex and queue implementations
+- Add UM.debug method
+- Implement Fiber scheduler (WIP)
+
+# 0.19.1 2025-11-03
 
 - Add `RB_GC_GUARD` in `process_runqueue_op`
 

data/TODO.md

@@ -1,44 +1,5 @@
 ## immediate
 
-- make a reproducer for segfault on timeout, spin lots of fibers where a timeout
-  wraps a #shift call (from an empty queue).
-- see also: https://mensfeld.pl/2025/11/ruby-ffi-gc-bug-hash-becomes-string/
-
-Analysis:
-
-- The segfault is related to timeouts
-- Looking at process_runqueue_op (um.c):
-
-  ```c
-  inline VALUE process_runqueue_op(struct um *machine, struct um_op *op) {
-    VALUE fiber = op->fiber;
-    VALUE value = op->value;
-
-    // on timeout, the op flags are changed to turn on OP_F_TRANSIENT
-    if (unlikely(op->flags & OP_F_TRANSIENT))
-      // here the op is freed, so the value is not visible to the GC anymoore
-      um_op_free(machine, op);
-
-    // if a GC occurs here, we risk a segfault
-
-    // value is used
-    return rb_fiber_transfer(fiber, 1, &value);
-  }
-  ```
-
-- So, a possible solution is to put a `RB_GC_GUARD` after the `return`.
-- But first, I want to be able to reproduce it. We can start by setting
-  `GC.stress = true` on tests and see if we segfault.
-
-## FiberScheduler implementation
-
-Some resources:
-
-- https://github.com/socketry/async/blob/main/context/getting-started.md
-- https://github.com/socketry/async/blob/main/context/scheduler.md
-- https://github.com/socketry/async/blob/main/lib/async/scheduler.rb#L28
-- 
-
 ## useful concurrency tools
 
 - debounce

data/examples/bm_fileno.rb

@@ -0,0 +1,33 @@
+# frozen_string_literal: true
+
+require 'bundler/inline'
+
+gemfile do
+  source 'https://rubygems.org'
+  gem 'benchmark'
+  gem 'benchmark-ips'
+end
+
+require 'benchmark/ips'
+
+r, w = IO.pipe
+
+class IO
+  def __fd__
+    @__fd__ ||= fileno
+  end
+end
+
+@map = ObjectSpace::WeakMap.new
+
+def cached_fileno(r)
+  @map[r] ||= r.fileno
+end
+
+Benchmark.ips do |x|
+  x.report('IO#fileno')       { r.fileno }
+  x.report('cached_fileno')   { cached_fileno(r) }
+  x.report('__fd__')          { r.__fd__ }
+
+  x.compare!(order: :baseline)
+end

data/examples/bm_mutex.rb

@@ -0,0 +1,85 @@
+# frozen_string_literal: true
+
+require 'bundler/inline'
+
+gemfile do
+  source 'https://rubygems.org'
+  gem 'uringmachine', path: '..'
+  gem 'benchmark'
+  gem 'benchmark-ips'
+end
+
+require 'benchmark/ips'
+require 'uringmachine'
+require 'thread'
+
+COUNT = 10
+CONCURRENCY = (ENV['CONCURRENCY'] || 4).to_i
+
+p(CONCURRENCY: CONCURRENCY)
+
+STRING = "foo...bar"
+REGEXP = /foo(.+)bar/
+
+def threads_setup
+  @threads_mutex = Mutex.new
+  @threads_start = Queue.new
+  @threads_done = Queue.new
+  @threads = CONCURRENCY.times.map {
+    Thread.new do
+      loop do
+        @threads_start.shift
+        COUNT.times do
+          @threads_mutex.synchronize do
+            COUNT.times do
+              STRING.match(REGEXP)[1]
+            end
+          end
+        end
+        @threads_done << true
+      end
+    end
+  }
+end
+
+def threads_run
+  @threads_start << true
+  @threads_done.shift
+end
+
+def um_setup
+  @um_machine = UM.new
+  @um_mutex = UM::Mutex.new
+  @um_start = UM::Queue.new
+  @um_done = UM::Queue.new
+  @fibers = CONCURRENCY.times.map {
+    @um_machine.spin do
+      loop do
+        @um_machine.shift(@um_start)
+        COUNT.times do
+          @um_machine.synchronize(@um_mutex) do
+            COUNT.times do
+              STRING.match(REGEXP)[1]
+            end
+          end
+        end
+        @um_machine.push(@um_done, true)
+      end
+    end
+  }
+end
+
+def um_run
+  @um_machine.push(@um_start, true)
+  @um_machine.shift(@um_done)
+end
+
+threads_setup
+um_setup
+
+Benchmark.ips do |x|
+  x.report("threads") { @threads_run }
+  x.report("UM")      { @um_run }
+
+  x.compare!(order: :baseline)
+end

data/examples/bm_mutex_single.rb

@@ -0,0 +1,33 @@
+# frozen_string_literal: true
+
+require 'bundler/inline'
+
+gemfile do
+  source 'https://rubygems.org'
+  gem 'uringmachine', path: '..'
+  gem 'benchmark'
+  gem 'benchmark-ips'
+end
+
+require 'benchmark/ips'
+require 'uringmachine'
+require 'thread'
+
+def threads_setup
+  @threads_mutex = Mutex.new
+end
+
+def um_setup
+  @um_machine = UM.new
+  @um_mutex = UM::Mutex.new
+end
+
+threads_setup
+um_setup
+
+Benchmark.ips do |x|
+  x.report("threads") { @threads_mutex.synchronize { } }
+  x.report("UM")      { @um_machine.synchronize(@um_mutex) {} }
+
+  x.compare!(order: :baseline)
+end

data/examples/bm_queue.rb

@@ -5,24 +5,26 @@ require 'bundler/inline'
 gemfile do
   source 'https://rubygems.org'
   gem 'uringmachine', path: '..'
+  gem 'benchmark'
   gem 'benchmark-ips'
 end
 
 require 'benchmark/ips'
-require 'uringmachine'
+require_relative '../lib/uringmachine'
 
 COUNT = 1000
-NUM_PRODUCERS = 2
-NUM_CONSUMERS = 10
+NUM_PRODUCERS = 8
+NUM_CONSUMERS = 8
+
+@queue = Queue.new
+@done = Queue.new
 
 def run_threads
-  queue = Queue.new
-  done = Queue.new
 
   NUM_PRODUCERS.times do
     Thread.new do
-      COUNT.times { queue << rand(1000) }
-      done << true
+      COUNT.times { @queue << rand(1000) }
+      @done << true
     end
   end
 
@@ -30,65 +32,64 @@ def run_threads
   NUM_CONSUMERS.times do
     Thread.new do
       loop do
-        item = queue.shift
+        item = @queue.shift
         break if item.nil?
 
         total += item
       end
-      done << true
+      @done << true
     end
   end
 
   # wait for producers
-  NUM_PRODUCERS.times { done.shift }
+  NUM_PRODUCERS.times { @done.shift }
 
   # stop and wait for consumers
   NUM_CONSUMERS.times do
-    queue << nil
-    done.shift
+    @queue << nil
+    @done.shift
   end
 
   total
 end
 
-def run_um
-  machine = UM.new
-  queue = UM::Queue.new
-  done = UM::Queue.new
+@machine = UM.new
+@um_queue = UM::Queue.new
+@um_done = UM::Queue.new
 
+def run_um
   NUM_PRODUCERS.times do
-    machine.spin do
-      COUNT.times { machine.push(queue, rand(1000)) }
-      machine.push(done, true)
+    @machine.spin do
+      COUNT.times { @machine.push(@um_queue, rand(1000)) }
+      @machine.push(@um_done, true)
     end
   end
 
   total = 0
   NUM_CONSUMERS.times do
-    machine.spin do
+    @machine.spin do
       loop do
-        item = machine.shift(queue)
+        item = @machine.shift(@um_queue)
         break if item.nil?
 
         total += item
       end
-      machine.push(done, true)
+      @machine.push(@um_done, true)
     end
   end
 
   # wait for producers
-  NUM_PRODUCERS.times { machine.shift(done) }
+  NUM_PRODUCERS.times { @machine.shift(@um_done) }
 
   # stop and wait for consumers
   NUM_CONSUMERS.times do
-    machine.push(queue, nil)
-    machine.shift(done)
+    @machine.push(@um_queue, nil)
+    @machine.shift(@um_done)
   end
 
   total
 end
 
-
 # puts "running"
 # res = run_threads
 # p threads: res
@@ -98,14 +99,13 @@ end
 #   p fibers: res
 # }
 
-
 # __END__
 
 Benchmark.ips do |x|
-  x.config(:time => 5, :warmup => 2)
+  x.config(:time => 10, :warmup => 3)
 
   x.report("threads") { run_threads }
   x.report("UM")      { run_um }
 
-  x.compare!
+  x.compare!(order: :baseline)
 end

data/examples/bm_send.rb

@@ -71,10 +71,6 @@ def um_send_bundle
   @machine.send_bundle(@server_fd, @bgid, @parts)
 end
 
-# um_send_bundle
-# 10.times { @machine.snooze }
-# exit
-
 p(STR_COUNT:, STR_SIZE:)
 
 Benchmark.ips do |x|
@@ -82,5 +78,6 @@ Benchmark.ips do |x|
   x.report('UM#write')       { um_write }
   x.report('UM#send')        { um_send }
   x.report('UM#send_bundle') { um_send_bundle }
-  x.compare!
+
+  x.compare!(order: :baseline)
 end

data/examples/bm_snooze.rb

@@ -15,37 +15,19 @@ ITERATIONS = 1000
 
 $machine = UringMachine.new
 
-def run_snooze
+def run_snooze_um
   count = 0
-  main = Fiber.current
 
-  f1 = Fiber.new do
-    loop do
-      count += 1
-      if count == ITERATIONS
-        $machine.schedule(main, nil)
-        break
-      else
-        $machine.snooze
-      end
-    end
-  end
+  f1 = $machine.spin {
+    ITERATIONS.times { count += 1; $machine.snooze }
+  }
+  f2 = $machine.spin {
+    ITERATIONS.times { count += 1; $machine.snooze }
+  }
 
-  f2 = Fiber.new do
-    loop do
-      count += 1
-      if count == ITERATIONS
-        $machine.schedule(main, nil)
-        break
-      else
-        $machine.snooze
-      end
-    end
-  end
+  $machine.join(f1, f2)
 
-  $machine.schedule(f1, nil)
-  $machine.schedule(f2, nil)
-  $machine.yield
+  count
 end
 
 def run_raw_transfer
@@ -53,36 +35,32 @@ def run_raw_transfer
   main = Fiber.current
   f2 = nil
   f1 = Fiber.new do
-    loop do
+    ITERATIONS.times do
       count += 1
-      if count == ITERATIONS
-        main.transfer(nil)
-        break
-      else
-        f2.transfer(nil)
-      end
+      f2.transfer(nil)
     end
   end
 
   f2 = Fiber.new do
-    loop do
+    ITERATIONS.times do
       count += 1
-      if count == ITERATIONS
-        main.transfer(nil)
-        break
-      else
-        f1.transfer(nil)
-      end
+      f1.transfer(nil)
     end
+    main.transfer(nil)
   end
 
   f1.transfer(nil)
+
+  count
 end
 
+p run_snooze_um:    run_snooze_um
+p run_raw_transfer: run_raw_transfer
+
 bm = Benchmark.ips do |x|
   x.config(:time => 5, :warmup => 2)
 
-  x.report("snooze")        { run_snooze }
+  x.report("snooze_um")     { run_snooze_um }
   x.report("raw transfer")  { run_raw_transfer }
 
   x.compare!

data/examples/bm_write.rb

@@ -11,7 +11,7 @@ end
 require 'benchmark'
 require 'uringmachine'
 
-ITERATIONS = 100000
+ITERATIONS = 10000
 BUF = ('*' * 8192).freeze
 FN = '/tmp/bm_write'
 
@@ -47,6 +47,9 @@ ensure
   fio.close
 end
 
+run_io_write(1)
+run_um_write(1)
+
 Benchmark.bm do |x|
   [1, 2, 4, 8].each do |c|
     x.report("IO (#{c} threads)")     { run_io_write(c) }

data/examples/fiber_scheduler_demo.rb

@@ -1,71 +1,35 @@
 # frozen_string_literal: true
 
-require_relative '../lib/uringmachine'
-require_relative '../lib/uringmachine/fiber_scheduler'
+require 'bundler/setup'
+
+require 'uringmachine'
+require 'uringmachine/fiber_scheduler'
 require 'net/http'
 
-machine = UringMachine.new
-scheduler = UM::FiberScheduler.new(machine)
+$machine = UringMachine.new
+scheduler = UM::FiberScheduler.new($machine)
 Fiber.set_scheduler scheduler
 
 i, o = IO.pipe
 
 f1 = Fiber.schedule do
-  # sleep 0.4
-  # o.write 'Hello, world!'
-  # o.close
+  puts "Hello from fiber 1!!!"
 end
 
 f2 = Fiber.schedule do
-  # puts "hi"
-  # 10.times do
-  #   sleep 0.1
-  #   puts "."
-  # end
+  puts "Hello from fiber 2!!!"
 end
 
-# Fiber.schedule do
-#   scheduler.block(:wait)
-# end
-
 f3 = Fiber.schedule do
-#   message = i.read
-#   puts message
-# rescue => e
-#   scheduler.p e
-#   scheduler.p e.backtrace
+  o.write 'hello'
+  puts "wrote it!"
+  o.close
 end
 
 f4 = Fiber.schedule do
-  puts '*' * 40
-  # tcp = TCPSocket.new('noteflakes.com', 80)
-  # tcp.timeout = 10
-  # scheduler.p tcp
-  # tcp << "GET / HTTP/1.1\r\nHost: noteflakes.com\r\n\r\n"
-  # s = tcp.read
-  # scheduler.p s: s
-  ret = Net::HTTP.get('noteflakes.com', '/ping')
-  scheduler.p ret: ret
-rescue => e
-  scheduler.p e
-  scheduler.p e.backtrace
+  str = i.read
+  puts "got: #{str}"
 end
 
-scheduler.join(f1, f2, f3, f4)
-
-__END__
-
-stdout_fd = STDOUT.fileno
-stdin_fd = STDIN.fileno
-machine.write(stdout_fd, "Hello, world!\n")
-
-loop do
-  machine.write(stdout_fd, "Say something: ")
-  buf = +''
-  res = machine.read(stdin_fd, buf, 8192)
-  if res > 0
-    machine.write(stdout_fd, "You said: #{buf}")
-  else
-    break
-  end
-end
+# scheduler.join
+# Fiber.set_scheduler(nil)

data/examples/fiber_scheduler_fork.rb

@@ -0,0 +1,24 @@
+# frozen_string_literal: true
+
+require 'bundler/setup'
+
+require 'uringmachine'
+require 'uringmachine/fiber_scheduler'
+require 'net/http'
+
+$machine = UringMachine.new
+scheduler = UM::FiberScheduler.new($machine)
+Fiber.set_scheduler scheduler
+
+p parent: Fiber.scheduler
+p Fiber.current
+
+pid = fork do
+  Fiber.scheduler.post_fork
+  Fiber.set_scheduler nil
+  p child: Fiber.scheduler
+  # Fiber.scheduler.post_fork
+end
+
+Process.wait(pid)
+p :done