uringmachine 0.3 → 0.5

data/vendor/liburing/.github/workflows/build.yml

@@ -7,7 +7,7 @@ on:
 
 jobs:
   build:
-    runs-on: ubuntu-22.04
+    runs-on: ubuntu-24.04
 
     strategy:
       fail-fast: false
@@ -19,6 +19,15 @@ jobs:
             cxx_pkg: g++-x86-64-linux-gnu
             cc: x86_64-linux-gnu-gcc
             cxx: x86_64-linux-gnu-g++
+            sanitize: 0
+
+          # x86-64 gcc asan
+          - arch: x86_64
+            cc_pkg: gcc-x86-64-linux-gnu
+            cxx_pkg: g++-x86-64-linux-gnu
+            cc: x86_64-linux-gnu-gcc
+            cxx: x86_64-linux-gnu-g++
+            sanitize: 1
 
           # x86-64 clang
           - arch: x86_64
@@ -28,6 +37,7 @@ jobs:
             cxx: clang++
             liburing_extra_flags: -Wshorten-64-to-32
             extra_flags: -Wmissing-prototypes -Wstrict-prototypes -Wunreachable-code-loop-increment -Wunreachable-code -Wmissing-variable-declarations -Wextra-semi-stmt
+            sanitize: 0
 
           # x86 (32-bit) gcc
           - arch: i686
@@ -35,6 +45,7 @@ jobs:
             cxx_pkg: g++-i686-linux-gnu
             cc: i686-linux-gnu-gcc
             cxx: i686-linux-gnu-g++
+            sanitize: 0
 
           # aarch64 gcc
           - arch: aarch64
@@ -42,6 +53,7 @@ jobs:
             cxx_pkg: g++-aarch64-linux-gnu
             cc: aarch64-linux-gnu-gcc
             cxx: aarch64-linux-gnu-g++
+            sanitize: 0
 
           # arm (32-bit) gcc
           - arch: arm
@@ -49,6 +61,7 @@ jobs:
             cxx_pkg: g++-arm-linux-gnueabi
             cc: arm-linux-gnueabi-gcc
             cxx: arm-linux-gnueabi-g++
+            sanitize: 0
 
           # riscv64
           - arch: riscv64
@@ -56,6 +69,7 @@ jobs:
             cxx_pkg: g++-riscv64-linux-gnu
             cc: riscv64-linux-gnu-gcc
             cxx: riscv64-linux-gnu-g++
+            sanitize: 0
 
           # powerpc64
           - arch: powerpc64
@@ -63,6 +77,7 @@ jobs:
             cxx_pkg: g++-powerpc64-linux-gnu
             cc: powerpc64-linux-gnu-gcc
             cxx: powerpc64-linux-gnu-g++
+            sanitize: 0
 
           # powerpc
           - arch: powerpc
@@ -70,6 +85,7 @@ jobs:
             cxx_pkg: g++-powerpc-linux-gnu
             cc: powerpc-linux-gnu-gcc
             cxx: powerpc-linux-gnu-g++
+            sanitize: 0
 
           # alpha
           - arch: alpha
@@ -77,6 +93,7 @@ jobs:
             cxx_pkg: g++-alpha-linux-gnu
             cc: alpha-linux-gnu-gcc
             cxx: alpha-linux-gnu-g++
+            sanitize: 0
 
           # mips64
           - arch: mips64
@@ -84,6 +101,7 @@ jobs:
             cxx_pkg: g++-mips64-linux-gnuabi64
             cc: mips64-linux-gnuabi64-gcc
             cxx: mips64-linux-gnuabi64-g++
+            sanitize: 0
 
           # mips
           - arch: mips
@@ -91,6 +109,7 @@ jobs:
             cxx_pkg: g++-mips-linux-gnu
             cc: mips-linux-gnu-gcc
             cxx: mips-linux-gnu-g++
+            sanitize: 0
 
           # hppa
           - arch: hppa
@@ -98,9 +117,11 @@ jobs:
             cxx_pkg: g++-hppa-linux-gnu
             cc: hppa-linux-gnu-gcc
             cxx: hppa-linux-gnu-g++
+            sanitize: 0
 
     env:
       FLAGS: -g -O3 -Wall -Wextra -Werror -Wno-sign-compare ${{matrix.extra_flags}}
+      SANITIZE: ${{matrix.sanitize}}
 
       # Flags for building sources in src/ dir only.
       LIBURING_CFLAGS: ${{matrix.liburing_extra_flags}}
@@ -128,10 +149,17 @@ jobs:
         ${{matrix.cxx}} --version;
 
     - name: Build
+      if: ${{matrix.sanitizer == '0'}}
       run: |
         ./configure --cc=${{matrix.cc}} --cxx=${{matrix.cxx}};
         make -j$(nproc) V=1 CPPFLAGS="-Werror" CFLAGS="$FLAGS" CXXFLAGS="$FLAGS";
 
+    - name: Build
+      if: ${{matrix.sanitizer == '1'}}
+      run: |
+        ./configure --cc=${{matrix.cc}} --cxx=${{matrix.cxx}} --enable-sanitizer;
+        make -j$(nproc) V=1 CPPFLAGS="-Werror" CFLAGS="$FLAGS" CXXFLAGS="$FLAGS";
+
     - name: Test install command
       run: |
         sudo make install;

data/vendor/liburing/.gitignore

@@ -26,11 +26,17 @@
 /examples/proxy
 /examples/send-zerocopy
 /examples/rsrc-update-bench
+/examples/kdigest
+/examples/reg-wait
 
 /test/*.t
 /test/*.dmesg
 /test/output/
 
+# Clang's compilation database file and directory.
+/.cache
+/compile_commands.json
+
 config-host.h
 config-host.mak
 config.log

data/vendor/liburing/CHANGELOG

@@ -1,3 +1,19 @@
+liburing-2.8 release
+- Add support for incrementally/partially consumed provided buffers,
+  usable with the provided buffer ring support.
+- Add support for foo_and_wait_min_timeout(), where it's possible to
+  define a minimum timeout for waiting to get batches of completions,
+  but if that fails, extend for a longer timeout without having any
+  extra context switches.
+- Add support for using different clock sources for completion waiting.
+- Great increase coverage of test cases, test case improvements and
+  fixes.
+- Man page updates
+- Don't leak _GNU_SOURCE via pkb-config --cflags
+- Support for address sanitizer
+- Add examples/kdigest sample program
+- Add discard helper, test, and man page
+
 liburing-2.7 release
 
 - Man page updates

data/vendor/liburing/CONTRIBUTING.md

@@ -0,0 +1,165 @@
+Introduction
+============
+
+liburing welcomes contributions, whether they be bug fixes, features, or
+documentation additions/updates. However, we do have some rules in place
+to govern the sanity of the project, and all contributions should follow
+the guidelines in this document. The main reasons for the rules are:
+
+1) Keep the code consistent
+2) Keep the git repository consistent
+3) Maintain bisectability
+
+Coding style
+============
+
+Generally, all the code in liburing should follow the same style. A few
+known exceptions exist, like syzbot test cases that got committed rather
+than re-writing them in a saner format. Any change you make, please
+follow the style of the code around you.
+
+Commit format
+=============
+
+Each commit should do one thing, and one thing only. If you find yourself,
+in the commit message, adding phrases like "Also do [...]" or "While in
+here [...]", then that's a sign that the change should have been split
+into multiple commits. If your change includes some refactoring of code to
+make your change possible, then that refactoring should be a separate
+commit, done first. That means this preparatory commit won't have any
+functional changes, and hence should be a no-op. It also means that your
+main commit, with the change that you actually care about, will be smaller
+and easier to review.
+
+Each commit must stand on its own in terms of what it provides, and how it
+works. Lots of changes are just a single commit, but for something a bit
+more involved, it's not uncommon to have a pull request contain multiple
+commits. Make each commit as simple as possible, and not any simpler. We'd
+much rather see 10 simple commits than 2 more complicated ones. If you
+stumble across something that needs fixing while making an unrelated
+change, then please make that change as a separate commit, explaining why
+it's being made.
+
+Each commit in a series must be buildable, it's not enough that the end
+result is buildable. See reason 3 in the introduction for why that's the
+case.
+
+No fixup commits! Sometimes people post a change and errors are pointed
+out in the commit, and the author then does a followup fix for that
+error. This isn't acceptable, please squash fixup commits into the
+commit that introduced the problem in the first place. This is done by
+amending the fix into the original commit that caused the issue. You can
+do that with git rebase -i <sha> and arrange the commit order such that
+the fixup is right after the original commit, and then use 's' (for
+squash) to squash the fixup into the original commit. Don't forget to
+edit the commit message while doing that, as git will combine the two
+commit messages into one. Or you can do it manually. Once done, force
+push your rewritten git history. See reasons 1-3 in the introduction
+series for why that is.
+
+Commit message
+==============
+
+A good commit explains the WHY of a commit - explain the reason for this
+commit to exist. Don't explain what the code in commit does, that should
+be readily apparent from just reading the code. If that isn't the case,
+then a comment in the code is going to be more useful than a lengthy
+explanation in the commit message. liburing commits use the following
+format:
+
+Title
+
+Body of commit
+
+Signed-off-by: ```My Identity <my@email.com>```
+
+That is, a descriptive title on the first line, then an empty line, then
+the body of the commit message, then an empty line, and finally an SOB
+tag. The signed-off-by exists to provide proof of origin, see the
+[DCO](https://developercertificate.org/).
+
+Example commit:
+
+```
+commit 0fe5c09195c0918f89582dd6ff098a58a0bdf62a
+Author: Jens Axboe <axboe@kernel.dk>
+Date:   Fri Sep 6 15:54:04 2024 -0600
+
+    configure: fix ublk_cmd header check
+    
+    The previous commit is mixing private structures and defines with public
+    uapi ones. Testing for UBLK_U_CMD_START_DEV is fine, CTRL_CMD_HAS_DATA
+    is not. And struct ublk_ctrl_cmd_data is not a public struct.
+    
+    Fixes: 83bc535a3118 ("configure: don't enable ublk if modern commands not available")
+    Signed-off-by: Jens Axboe <axboe@kernel.dk>
+```
+
+Since this change is pretty trivial, a huge explanation need not be given
+as to the reasonings for the change. However, for more complicated
+changes, better reasonings should be given.
+
+A Fixes line can be added if this commit fixes an issue in a previous
+commit. That kind of meta data can be useful down the line for finding
+dependencies between commits. Adding the following to your .gitconfig:
+
+```
+[pretty]
+	fixes = Fixes: %h (\"%s\")
+```
+
+and running ```git fixes <sha>``` will then generate the correctly
+formatted Fixes line for the commit. Likewise, other meta data can be:
+
+Link: https://somesite/somewhere
+
+can be useful to link to a discussion around the issue that led to this
+commit, perhaps a bug report. This can be a GitHub issue as well. If a
+commit closes/solves a GitHub issue, than:
+
+Closes: https://github.com/axboe/liburing/issues/XXXX
+
+can also be used.
+
+Each commit message should be formatted so each full line is 72-74 chars
+wide. For many of us, GitHub is not the primary location, and git log is
+often used in a terminal to browse the repo. Breaking lines at 72-74
+characters retains readability in an xterm/terminal.
+
+Pull Requests
+=============
+
+The git repository itself is the canonical location for information. It's
+quite fine to provide a lengthy explanation for a pull request on GitHub,
+however please ensure that this doesn't come at the expense of the commit
+messages themselves being lacking. The commit messages should stand on
+their own and contain everything that you'd otherwise put in the PR
+message. If you've worked on projects that send patches before, consider
+the PR message similar to the cover letter for a series of patches.
+
+Most contributors seem to use GH for sending patches, which is fine. If
+you prefer using email, then patches can also be sent to the io_uring
+mailing list: io-uring@vger.kernel.org.
+
+liburing doesn't squash/rebase-on-merge, or other heinous practices
+sometimes seen elsewhere. Whatever sha your commit has in your tree is
+what it'll have in the upstream tree. Patches are applied directly, and
+pull requests are merged with a merge commit. If meta data needs to go
+into the merge commit, then it will go into the merge commit message.
+This means that you don't need to continually rebase your changes on top
+of the master branch.
+
+Testing changes
+===============
+
+You should ALWAYS test your changes, no matter how trivial or obviously
+correct they may seem. Nobody is infallible, and making mistakes is only
+human.
+
+liburing contains a wide variety of functional tests. If you make changes
+to liburing, then you should run the test cases. This is done by building
+the repo and running ```make runtests```. Note that some of the liburing
+tests test for defects in older kernels, and hence it's possible that they
+will crash on an outdated kernel that doesn't contain fixes from the
+stable kernel tree. If in doubt, building and running the tests in a vm is
+encouraged.

data/vendor/liburing/configure

@@ -28,6 +28,8 @@ for opt do
   ;;
   --use-libc) use_libc=yes
   ;;
+  --enable-sanitizer) use_sanitizer=yes
+  ;;
   *)
     echo "ERROR: unknown option $opt"
     echo "Try '$0 --help' for more information"
@@ -76,6 +78,7 @@ Options: [defaults in brackets after descriptions]
   --cc=CMD                 use CMD as the C compiler
   --cxx=CMD                use CMD as the C++ compiler
   --use-libc               use libc for liburing (useful for hardening)
+  --enable-sanitizer       compile liburing with the address and undefined behaviour sanitizers. (useful for debugging)
 EOF
 exit 0
 fi
@@ -414,6 +417,21 @@ if compile_prog "" "" "futexv"; then
 fi
 print_config "futex waitv support" "$futexv"
 
+##########################################
+# Check block discard cmd support
+discard_cmd="no"
+cat > $TMPC << EOF
+#include <linux/blkdev.h>
+int main(void)
+{
+  return BLOCK_URING_CMD_DISCARD;
+}
+EOF
+if compile_prog "" "" "discard command"; then
+  discard_cmd="yes"
+fi
+print_config "io_uring discard command support" "$discard_cmd"
+
 ##########################################
 # Check idtype_t support
 has_idtype_t="no"
@@ -471,6 +489,26 @@ fi
 print_config "has_fanotify" "$has_fanotify"
 ####################################################
 
+##########################################
+# check for ublk headers
+ublk_header="no"
+cat > $TMPC << EOF
+#include <string.h>
+#include <sys/ioctl.h>
+#include <linux/ublk_cmd.h>
+int main(int argc, char **argv)
+{
+  struct ublksrv_ctrl_cmd cmd = { };
+
+  cmd.addr = UBLK_U_CMD_START_DEV;
+  return cmd.queue_id;
+}
+EOF
+if compile_prog "" "" "ublk_header"; then
+  ublk_header="yes"
+fi
+print_config "ublk_header" "$ublk_header"
+
 if test "$liburing_nolibc" = "yes"; then
   output_sym "CONFIG_NOLIBC"
 fi
@@ -510,6 +548,15 @@ fi
 if test "$futexv" = "yes"; then
   output_sym "CONFIG_HAVE_FUTEXV"
 fi
+if test "$ublk_header" = "yes"; then
+  output_sym "CONFIG_HAVE_UBLK_HEADER"
+fi
+if test "$use_sanitizer" = "yes"; then
+  output_sym "CONFIG_USE_SANITIZER"
+  print_config "use sanitizer" "yes"
+else
+  print_config "use sanitizer" "no"
+fi
 
 echo "CC=$cc" >> $config_host_mak
 print_config "CC" "$cc"
@@ -619,6 +666,23 @@ typedef enum
 } idtype_t;
 EOF
 fi
+
+if test "$discard_cmd" != "yes"; then
+cat >> $compat_h << EOF
+
+#include <linux/ioctl.h>
+
+#ifndef BLOCK_URING_CMD_DISCARD
+#define BLOCK_URING_CMD_DISCARD                        _IO(0x12, 0)
+#endif
+
+EOF
+else cat >> $compat_h << EOF
+#include <linux/blkdev.h>
+
+EOF
+fi
+
 cat >> $compat_h << EOF
 #endif
 EOF

data/vendor/liburing/examples/Makefile

@@ -13,6 +13,12 @@ endif
 LDFLAGS ?=
 override LDFLAGS += -L../src/ -luring -lpthread
 
+ifeq ($(CONFIG_USE_SANITIZER),y)
+	override CFLAGS += -fsanitize=address,undefined -fno-omit-frame-pointer -fno-optimize-sibling-calls
+	override CPPFLAGS += -fsanitize=address,undefined -fno-omit-frame-pointer -fno-optimize-sibling-calls
+	override LDFLAGS += -fsanitize=address,undefined
+endif
+
 example_srcs := \
 	io_uring-close-test.c \
 	io_uring-cp.c \
@@ -22,9 +28,11 @@ example_srcs := \
 	napi-busy-poll-client.c \
 	napi-busy-poll-server.c \
 	poll-bench.c \
+	reg-wait.c \
 	send-zerocopy.c \
 	rsrc-update-bench.c \
-	proxy.c
+	proxy.c \
+	kdigest.c
 
 all_targets :=