uringmachine 0.3 → 0.4

data/test/test_um.rb

@@ -160,7 +160,9 @@ end
 class SleepTest < UMBaseTest
   def test_sleep
     t0 = monotonic_clock
+    assert_equal 0, machine.pending_count
     res = machine.sleep(0.1)
+    assert_equal 0, machine.pending_count
     t1 = monotonic_clock
     assert_in_range 0.09..0.13, t1 - t0
     assert_equal 0.1, res
@@ -173,7 +175,9 @@ class ReadTest < UMBaseTest
     w << 'foobar'
 
     buf = +''
+    assert_equal 0, machine.pending_count
     res = machine.read(r.fileno, buf, 3)
+    assert_equal 0, machine.pending_count
     assert_equal 3, res
     assert_equal 'foo', buf
 
@@ -339,7 +343,9 @@ class WriteTest < UMBaseTest
   def test_write
     r, w = IO.pipe
 
+    assert_equal 0, machine.pending_count
     machine.write(w.fileno, 'foo')
+    assert_equal 0, machine.pending_count
     assert_equal 'foo', r.readpartial(3)
 
     machine.write(w.fileno, 'bar', 2)
@@ -349,16 +355,33 @@ class WriteTest < UMBaseTest
   def test_write_bad_fd
     r, _w = IO.pipe
 
+    assert_equal 0, machine.pending_count
     assert_raises(Errno::EBADF) do
       machine.write(r.fileno, 'foo')
     end
+    assert_equal 0, machine.pending_count
+  end
+end
+
+class Closetest < UMBaseTest
+  def test_close
+    r, w = IO.pipe
+    machine.write(w.fileno, 'foo')
+    assert_equal 'foo', r.readpartial(3)
+
+    assert_equal 0, machine.pending_count
+    machine.close(w.fileno)
+    assert_equal 0, machine.pending_count
+    assert_equal '', r.read
+
+    assert_raises(Errno::EBADF) { machine.close(w.fileno) }
   end
 end
 
 class AcceptTest < UMBaseTest
   def setup
     super
-    @port = 9000 + rand(1000)
+    @port = assign_port
     @server = TCPServer.open('127.0.0.1', @port)
   end
 
@@ -370,7 +393,9 @@ class AcceptTest < UMBaseTest
   def test_accept
     conn = TCPSocket.new('127.0.0.1', @port)
     
+    assert_equal 0, machine.pending_count
     fd = machine.accept(@server.fileno)
+    assert_equal 0, machine.pending_count
     assert_kind_of Integer, fd
     assert fd > 0
 
@@ -384,7 +409,7 @@ end
 class AcceptEachTest < UMBaseTest
   def setup
     super
-    @port = 9000 + rand(1000)
+    @port = assign_port
     @server = TCPServer.open('127.0.0.1', @port)
   end
 
@@ -394,11 +419,15 @@ class AcceptEachTest < UMBaseTest
   end
 
   def test_accept_each
-    conns = 3.times.map { TCPSocket.new('127.0.0.1', @port) }
+    conns = []
+    t = Thread.new do
+      sleep 0.1
+      3.times { conns << TCPSocket.new('127.0.0.1', @port) }
+    end
 
     count = 0
     machine.accept_each(@server.fileno) do |fd|
-      machine.write(fd, (count += 1).to_s)
+      count += 1
       break if count == 3
     end
 
@@ -406,9 +435,200 @@ class AcceptEachTest < UMBaseTest
     assert_equal 1, machine.pending_count
     machine.snooze
     assert_equal 0, machine.pending_count
+  ensure
+    t&.kill
+  end
+end
+
+class SocketTest < UMBaseTest
+  def test_socket
+    assert_equal 0, machine.pending_count
+    fd = machine.socket(UM::AF_INET, UM::SOCK_DGRAM, 0, 0);
+    assert_equal 0, machine.pending_count
+    assert_kind_of Integer, fd
+    assert fd > 0
+
+    assert_raises(Errno::EDESTADDRREQ) { machine.write(fd, 'foo') }
+  end
+end
+
+class ConnectTest < UMBaseTest
+  def setup
+    super
+    @port = assign_port
+    @server = TCPServer.open('127.0.0.1', @port)
+  end
+
+  def teardown
+    @server&.close
+    super
+  end
+
+  def test_connect
+    t = Thread.new do
+      conn = @server.accept
+      conn.write('foobar')
+      sleep
+    end
+
+    fd = machine.socket(UM::AF_INET, UM::SOCK_STREAM, 0, 0);
+    assert_equal 0, machine.pending_count
+    res = machine.connect(fd, '127.0.0.1', @port)
+    assert_equal 0, machine.pending_count
+    assert_equal 0, res
+
+    buf = +''
+    res = machine.read(fd, buf, 42)
+    assert_equal 6, res
+    assert_equal 'foobar', buf
+  ensure
+    t&.kill
+  end
 
-    assert_equal '1', conns[0].readpartial(3)
-    assert_equal '2', conns[1].readpartial(3)
-    assert_equal '3', conns[2].readpartial(3)
+  def test_connect_with_bad_addr
+    fd = machine.socket(UM::AF_INET, UM::SOCK_STREAM, 0, 0);
+    assert_equal 0, machine.pending_count
+    assert_raises(Errno::ENETUNREACH) { machine.connect(fd, 'a.b.c.d', @port) }
+    assert_equal 0, machine.pending_count
   end
 end
+
+class SendTest < UMBaseTest
+  def setup
+    super
+    @port = assign_port
+    @server = TCPServer.open('127.0.0.1', @port)
+  end
+
+  def teardown
+    @server&.close
+    super
+  end
+
+  def test_send
+    t = Thread.new do
+      conn = @server.accept
+      str = conn.readpartial(42)
+      conn.write("You said: #{str}")
+      sleep
+    end
+
+    fd = machine.socket(UM::AF_INET, UM::SOCK_STREAM, 0, 0);
+    res = machine.connect(fd, '127.0.0.1', @port)
+    assert_equal 0, res
+
+    res = machine.send(fd, 'foobar', 6, 0)
+    assert_equal 6, res
+
+    buf = +''
+    res = machine.read(fd, buf, 42)
+    assert_equal 16, res
+    assert_equal 'You said: foobar', buf
+  ensure
+    t&.kill
+  end
+end
+
+class RecvTest < UMBaseTest
+  def setup
+    super
+    @port = assign_port
+    @server = TCPServer.open('127.0.0.1', @port)
+  end
+
+  def teardown
+    @server&.close
+    super
+  end
+
+  def test_recv
+    t = Thread.new do
+      conn = @server.accept
+      conn.write('foobar')
+      sleep
+    end
+
+    fd = machine.socket(UM::AF_INET, UM::SOCK_STREAM, 0, 0);
+    res = machine.connect(fd, '127.0.0.1', @port)
+    assert_equal 0, res
+
+    buf = +''
+    res = machine.recv(fd, buf, 42, 0)
+    assert_equal 6, res
+    assert_equal 'foobar', buf
+  ensure
+    t&.kill
+  end
+end
+
+class BindTest < UMBaseTest
+  def setup
+    super
+    @port = assign_port
+  end
+
+  def test_bind
+    assert_equal 0, machine.pending_count
+    fd = machine.socket(UM::AF_INET, UM::SOCK_DGRAM, 0, 0);
+    res = machine.bind(fd, '127.0.0.1', @port)
+    assert_equal 0, res
+    assert_equal 0, machine.pending_count
+
+    peer = UDPSocket.new
+    peer.connect('127.0.0.1', @port)
+    peer.send 'foo', 0
+
+    buf = +''
+    res = machine.recv(fd, buf, 8192, 0)
+    assert_equal 3, res
+    assert_equal 'foo', buf
+  end
+
+  def test_bind_invalid_args
+    assert_equal 0, machine.pending_count
+
+    fd = machine.socket(UM::AF_INET, UM::SOCK_DGRAM, 0, 0);
+    assert_raises(Errno::EACCES) { machine.bind(fd, 'foo.bar.baz', 3) }
+    assert_raises(Errno::EBADF) { machine.bind(-3, '127.0.01', 1234) }
+
+    assert_equal 0, machine.pending_count
+  end
+end
+
+class ListenTest < UMBaseTest
+  def setup
+    super
+    @port = assign_port
+  end
+
+  def test_listen
+    fd = machine.socket(UM::AF_INET, UM::SOCK_STREAM, 0, 0);
+    machine.bind(fd, '127.0.0.1', @port)
+    res = machine.listen(fd, 5)
+    assert_equal 0, res
+    assert_equal 0, machine.pending_count
+
+    conn = nil
+    t = Thread.new do
+      sleep 0.01
+      conn = TCPSocket.new('127.0.0.1', @port)
+    end
+
+    conn_fd = machine.accept(fd)
+    t.join
+    assert_kind_of TCPSocket, conn
+
+    machine.send(conn_fd, 'foo', 3, 0)
+
+    buf = conn.readpartial(42)
+    assert_equal 'foo', buf
+  ensure
+    t&.kill
+  end
+end
+
+class ConstTest < UMBaseTest
+  def test_constants
+    assert_equal UM::SOCK_STREAM, UM::SOCK_STREAM
+  end
+end

data/vendor/liburing/.github/workflows/build.yml

@@ -7,7 +7,7 @@ on:
 
 jobs:
   build:
-    runs-on: ubuntu-22.04
+    runs-on: ubuntu-24.04
 
     strategy:
       fail-fast: false
@@ -19,6 +19,15 @@ jobs:
             cxx_pkg: g++-x86-64-linux-gnu
             cc: x86_64-linux-gnu-gcc
             cxx: x86_64-linux-gnu-g++
+            sanitize: 0
+
+          # x86-64 gcc asan
+          - arch: x86_64
+            cc_pkg: gcc-x86-64-linux-gnu
+            cxx_pkg: g++-x86-64-linux-gnu
+            cc: x86_64-linux-gnu-gcc
+            cxx: x86_64-linux-gnu-g++
+            sanitize: 1
 
           # x86-64 clang
           - arch: x86_64
@@ -28,6 +37,7 @@ jobs:
             cxx: clang++
             liburing_extra_flags: -Wshorten-64-to-32
             extra_flags: -Wmissing-prototypes -Wstrict-prototypes -Wunreachable-code-loop-increment -Wunreachable-code -Wmissing-variable-declarations -Wextra-semi-stmt
+            sanitize: 0
 
           # x86 (32-bit) gcc
           - arch: i686
@@ -35,6 +45,7 @@ jobs:
             cxx_pkg: g++-i686-linux-gnu
             cc: i686-linux-gnu-gcc
             cxx: i686-linux-gnu-g++
+            sanitize: 0
 
           # aarch64 gcc
           - arch: aarch64
@@ -42,6 +53,7 @@ jobs:
             cxx_pkg: g++-aarch64-linux-gnu
             cc: aarch64-linux-gnu-gcc
             cxx: aarch64-linux-gnu-g++
+            sanitize: 0
 
           # arm (32-bit) gcc
           - arch: arm
@@ -49,6 +61,7 @@ jobs:
             cxx_pkg: g++-arm-linux-gnueabi
             cc: arm-linux-gnueabi-gcc
             cxx: arm-linux-gnueabi-g++
+            sanitize: 0
 
           # riscv64
           - arch: riscv64
@@ -56,6 +69,7 @@ jobs:
             cxx_pkg: g++-riscv64-linux-gnu
             cc: riscv64-linux-gnu-gcc
             cxx: riscv64-linux-gnu-g++
+            sanitize: 0
 
           # powerpc64
           - arch: powerpc64
@@ -63,6 +77,7 @@ jobs:
             cxx_pkg: g++-powerpc64-linux-gnu
             cc: powerpc64-linux-gnu-gcc
             cxx: powerpc64-linux-gnu-g++
+            sanitize: 0
 
           # powerpc
           - arch: powerpc
@@ -70,6 +85,7 @@ jobs:
             cxx_pkg: g++-powerpc-linux-gnu
             cc: powerpc-linux-gnu-gcc
             cxx: powerpc-linux-gnu-g++
+            sanitize: 0
 
           # alpha
           - arch: alpha
@@ -77,6 +93,7 @@ jobs:
             cxx_pkg: g++-alpha-linux-gnu
             cc: alpha-linux-gnu-gcc
             cxx: alpha-linux-gnu-g++
+            sanitize: 0
 
           # mips64
           - arch: mips64
@@ -84,6 +101,7 @@ jobs:
             cxx_pkg: g++-mips64-linux-gnuabi64
             cc: mips64-linux-gnuabi64-gcc
             cxx: mips64-linux-gnuabi64-g++
+            sanitize: 0
 
           # mips
           - arch: mips
@@ -91,6 +109,7 @@ jobs:
             cxx_pkg: g++-mips-linux-gnu
             cc: mips-linux-gnu-gcc
             cxx: mips-linux-gnu-g++
+            sanitize: 0
 
           # hppa
           - arch: hppa
@@ -98,9 +117,11 @@ jobs:
             cxx_pkg: g++-hppa-linux-gnu
             cc: hppa-linux-gnu-gcc
             cxx: hppa-linux-gnu-g++
+            sanitize: 0
 
     env:
       FLAGS: -g -O3 -Wall -Wextra -Werror -Wno-sign-compare ${{matrix.extra_flags}}
+      SANITIZE: ${{matrix.sanitize}}
 
       # Flags for building sources in src/ dir only.
       LIBURING_CFLAGS: ${{matrix.liburing_extra_flags}}
@@ -128,10 +149,17 @@ jobs:
         ${{matrix.cxx}} --version;
 
     - name: Build
+      if: ${{matrix.sanitizer == '0'}}
       run: |
         ./configure --cc=${{matrix.cc}} --cxx=${{matrix.cxx}};
         make -j$(nproc) V=1 CPPFLAGS="-Werror" CFLAGS="$FLAGS" CXXFLAGS="$FLAGS";
 
+    - name: Build
+      if: ${{matrix.sanitizer == '1'}}
+      run: |
+        ./configure --cc=${{matrix.cc}} --cxx=${{matrix.cxx}} --enable-sanitizer;
+        make -j$(nproc) V=1 CPPFLAGS="-Werror" CFLAGS="$FLAGS" CXXFLAGS="$FLAGS";
+
     - name: Test install command
       run: |
         sudo make install;

data/vendor/liburing/.gitignore

@@ -26,6 +26,7 @@
 /examples/proxy
 /examples/send-zerocopy
 /examples/rsrc-update-bench
+/examples/kdigest
 
 /test/*.t
 /test/*.dmesg

data/vendor/liburing/CHANGELOG

@@ -1,3 +1,18 @@
+liburing-2.8 release
+- Add support for incrementally/partially consumed provided buffers,
+  usable with the provided buffer ring support.
+- Add support for foo_and_wait_min_timeout(), where it's possible to
+  define a minimum timeout for waiting to get batches of completions,
+  but if that fails, extend for a longer timeout without having any
+  extra context switches.
+- Add support for using different clock sources for completion waiting.
+- Great increase coverage of test cases, test case improvements and
+  fixes.
+- Man page updates
+- Don't leak _GNU_SOURCE via pkb-config --cflags
+- Support for address sanitizer
+- Add examples/kdigest sample program
+
 liburing-2.7 release
 
 - Man page updates

data/vendor/liburing/CONTRIBUTING.md

@@ -0,0 +1,165 @@
+Introduction
+============
+
+liburing welcomes contributions, whether they be bug fixes, features, or
+documentation additions/updates. However, we do have some rules in place
+to govern the sanity of the project, and all contributions should follow
+the guidelines in this document. The main reasons for the rules are:
+
+1) Keep the code consistent
+2) Keep the git repository consistent
+3) Maintain bisectability
+
+Coding style
+============
+
+Generally, all the code in liburing should follow the same style. A few
+known exceptions exist, like syzbot test cases that got committed rather
+than re-writing them in a saner format. Any change you make, please
+follow the style of the code around you.
+
+Commit format
+=============
+
+Each commit should do one thing, and one thing only. If you find yourself,
+in the commit message, adding phrases like "Also do [...]" or "While in
+here [...]", then that's a sign that the change should have been split
+into multiple commits. If your change includes some refactoring of code to
+make your change possible, then that refactoring should be a separate
+commit, done first. That means this preparatory commit won't have any
+functional changes, and hence should be a no-op. It also means that your
+main commit, with the change that you actually care about, will be smaller
+and easier to review.
+
+Each commit must stand on its own in terms of what it provides, and how it
+works. Lots of changes are just a single commit, but for something a bit
+more involved, it's not uncommon to have a pull request contain multiple
+commits. Make each commit as simple as possible, and not any simpler. We'd
+much rather see 10 simple commits than 2 more complicated ones. If you
+stumble across something that needs fixing while making an unrelated
+change, then please make that change as a separate commit, explaining why
+it's being made.
+
+Each commit in a series must be buildable, it's not enough that the end
+result is buildable. See reason 3 in the introduction for why that's the
+case.
+
+No fixup commits! Sometimes people post a change and errors are pointed
+out in the commit, and the author then does a followup fix for that
+error. This isn't acceptable, please squash fixup commits into the
+commit that introduced the problem in the first place. This is done by
+amending the fix into the original commit that caused the issue. You can
+do that with git rebase -i <sha> and arrange the commit order such that
+the fixup is right after the original commit, and then use 's' (for
+squash) to squash the fixup into the original commit. Don't forget to
+edit the commit message while doing that, as git will combine the two
+commit messages into one. Or you can do it manually. Once done, force
+push your rewritten git history. See reasons 1-3 in the introduction
+series for why that is.
+
+Commit message
+==============
+
+A good commit explains the WHY of a commit - explain the reason for this
+commit to exist. Don't explain what the code in commit does, that should
+be readily apparent from just reading the code. If that isn't the case,
+then a comment in the code is going to be more useful than a lengthy
+explanation in the commit message. liburing commits use the following
+format:
+
+Title
+
+Body of commit
+
+Signed-off-by: ```My Identity <my@email.com>```
+
+That is, a descriptive title on the first line, then an empty line, then
+the body of the commit message, then an empty line, and finally an SOB
+tag. The signed-off-by exists to provide proof of origin, see the
+[DCO](https://developercertificate.org/).
+
+Example commit:
+
+```
+commit 0fe5c09195c0918f89582dd6ff098a58a0bdf62a
+Author: Jens Axboe <axboe@kernel.dk>
+Date:   Fri Sep 6 15:54:04 2024 -0600
+
+    configure: fix ublk_cmd header check
+    
+    The previous commit is mixing private structures and defines with public
+    uapi ones. Testing for UBLK_U_CMD_START_DEV is fine, CTRL_CMD_HAS_DATA
+    is not. And struct ublk_ctrl_cmd_data is not a public struct.
+    
+    Fixes: 83bc535a3118 ("configure: don't enable ublk if modern commands not available")
+    Signed-off-by: Jens Axboe <axboe@kernel.dk>
+```
+
+Since this change is pretty trivial, a huge explanation need not be given
+as to the reasonings for the change. However, for more complicated
+changes, better reasonings should be given.
+
+A Fixes line can be added if this commit fixes an issue in a previous
+commit. That kind of meta data can be useful down the line for finding
+dependencies between commits. Adding the following to your .gitconfig:
+
+```
+[pretty]
+	fixes = Fixes: %h (\"%s\")
+```
+
+and running ```git fixes <sha>``` will then generate the correctly
+formatted Fixes line for the commit. Likewise, other meta data can be:
+
+Link: https://somesite/somewhere
+
+can be useful to link to a discussion around the issue that led to this
+commit, perhaps a bug report. This can be a GitHub issue as well. If a
+commit closes/solves a GitHub issue, than:
+
+Closes: https://github.com/axboe/liburing/issues/XXXX
+
+can also be used.
+
+Each commit message should be formatted so each full line is 72-74 chars
+wide. For many of us, GitHub is not the primary location, and git log is
+often used in a terminal to browse the repo. Breaking lines at 72-74
+characters retains readability in an xterm/terminal.
+
+Pull Requests
+=============
+
+The git repository itself is the canonical location for information. It's
+quite fine to provide a lengthy explanation for a pull request on GitHub,
+however please ensure that this doesn't come at the expense of the commit
+messages themselves being lacking. The commit messages should stand on
+their own and contain everything that you'd otherwise put in the PR
+message. If you've worked on projects that send patches before, consider
+the PR message similar to the cover letter for a series of patches.
+
+Most contributors seem to use GH for sending patches, which is fine. If
+you prefer using email, then patches can also be sent to the io_uring
+mailing list: io-uring@vger.kernel.org.
+
+liburing doesn't squash/rebase-on-merge, or other heinous practices
+sometimes seen elsewhere. Whatever sha your commit has in your tree is
+what it'll have in the upstream tree. Patches are applied directly, and
+pull requests are merged with a merge commit. If meta data needs to go
+into the merge commit, then it will go into the merge commit message.
+This means that you don't need to continually rebase your changes on top
+of the master branch.
+
+Testing changes
+===============
+
+You should ALWAYS test your changes, no matter how trivial or obviously
+correct they may seem. Nobody is infallible, and making mistakes is only
+human.
+
+liburing contains a wide variety of functional tests. If you make changes
+to liburing, then you should run the test cases. This is done by building
+the repo and running ```make runtests```. Note that some of the liburing
+tests test for defects in older kernels, and hence it's possible that they
+will crash on an outdated kernel that doesn't contain fixes from the
+stable kernel tree. If in doubt, building and running the tests in a vm is
+encouraged.

data/vendor/liburing/configure

@@ -28,6 +28,8 @@ for opt do
   ;;
   --use-libc) use_libc=yes
   ;;
+  --enable-sanitizer) use_sanitizer=yes
+  ;;
   *)
     echo "ERROR: unknown option $opt"
     echo "Try '$0 --help' for more information"
@@ -76,6 +78,7 @@ Options: [defaults in brackets after descriptions]
   --cc=CMD                 use CMD as the C compiler
   --cxx=CMD                use CMD as the C++ compiler
   --use-libc               use libc for liburing (useful for hardening)
+  --enable-sanitizer       compile liburing with the address and undefined behaviour sanitizers. (useful for debugging)
 EOF
 exit 0
 fi
@@ -471,6 +474,26 @@ fi
 print_config "has_fanotify" "$has_fanotify"
 ####################################################
 
+##########################################
+# check for ublk headers
+ublk_header="no"
+cat > $TMPC << EOF
+#include <string.h>
+#include <sys/ioctl.h>
+#include <linux/ublk_cmd.h>
+int main(int argc, char **argv)
+{
+  struct ublksrv_ctrl_cmd cmd = { };
+
+  cmd.addr = UBLK_U_CMD_START_DEV;
+  return cmd.queue_id;
+}
+EOF
+if compile_prog "" "" "ublk_header"; then
+  ublk_header="yes"
+fi
+print_config "ublk_header" "$ublk_header"
+
 if test "$liburing_nolibc" = "yes"; then
   output_sym "CONFIG_NOLIBC"
 fi
@@ -510,6 +533,15 @@ fi
 if test "$futexv" = "yes"; then
   output_sym "CONFIG_HAVE_FUTEXV"
 fi
+if test "$ublk_header" = "yes"; then
+  output_sym "CONFIG_HAVE_UBLK_HEADER"
+fi
+if test "$use_sanitizer" = "yes"; then
+  output_sym "CONFIG_USE_SANITIZER"
+  print_config "use sanitizer" "yes"
+else
+  print_config "use sanitizer" "no"
+fi
 
 echo "CC=$cc" >> $config_host_mak
 print_config "CC" "$cc"

data/vendor/liburing/examples/Makefile

@@ -13,6 +13,12 @@ endif
 LDFLAGS ?=
 override LDFLAGS += -L../src/ -luring -lpthread
 
+ifeq ($(CONFIG_USE_SANITIZER),y)
+	override CFLAGS += -fsanitize=address,undefined -fno-omit-frame-pointer -fno-optimize-sibling-calls
+	override CPPFLAGS += -fsanitize=address,undefined -fno-omit-frame-pointer -fno-optimize-sibling-calls
+	override LDFLAGS += -fsanitize=address,undefined
+endif
+
 example_srcs := \
 	io_uring-close-test.c \
 	io_uring-cp.c \
@@ -24,7 +30,8 @@ example_srcs := \
 	poll-bench.c \
 	send-zerocopy.c \
 	rsrc-update-bench.c \
-	proxy.c
+	proxy.c \
+	kdigest.c
 
 all_targets :=