polyphony 0.94 → 0.96
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.github/workflows/test.yml +3 -3
- data/.github/workflows/test_io_uring.yml +4 -4
- data/.gitignore +3 -3
- data/CHANGELOG.md +19 -0
- data/docs/api-reference/fiber.md +2 -2
- data/docs/api-reference/object.md +3 -3
- data/docs/main-concepts/exception-handling.md +2 -2
- data/examples/adapters/redis_blpop.rb +4 -3
- data/examples/adapters/redis_channels.rb +16 -7
- data/examples/core/await.rb +1 -1
- data/examples/io/readline.rb +19 -0
- data/ext/polyphony/backend_common.c +25 -3
- data/ext/polyphony/backend_io_uring.c +18 -16
- data/ext/polyphony/backend_libev.c +2 -2
- data/ext/polyphony/event.c +1 -1
- data/ext/polyphony/extconf.rb +5 -3
- data/ext/polyphony/fiber.c +5 -13
- data/ext/polyphony/io_extensions.c +1 -1
- data/ext/polyphony/pipe.c +1 -1
- data/ext/polyphony/polyphony.c +1 -1
- data/ext/polyphony/polyphony_ext.c +1 -1
- data/ext/polyphony/queue.c +1 -1
- data/ext/polyphony/ring_buffer.c +1 -0
- data/ext/polyphony/socket_extensions.c +1 -1
- data/ext/polyphony/thread.c +1 -1
- data/lib/polyphony/adapters/readline.rb +6 -4
- data/lib/polyphony/adapters/redis.rb +28 -87
- data/lib/polyphony/core/channel.rb +15 -0
- data/lib/polyphony/core/sync.rb +4 -0
- data/lib/polyphony/debugger.rb +2 -2
- data/lib/polyphony/extensions/socket.rb +2 -0
- data/lib/polyphony/version.rb +1 -1
- data/lib/polyphony.rb +4 -0
- data/polyphony.gemspec +10 -10
- data/test/helper.rb +0 -5
- data/test/test_backend.rb +5 -1
- data/test/test_enumerator.rb +46 -0
- data/test/test_ext.rb +63 -0
- data/test/test_io.rb +241 -216
- data/test/test_socket.rb +1 -1
- data/test/test_thread_pool.rb +5 -5
- data/vendor/liburing/.github/workflows/build.yml +51 -5
- data/vendor/liburing/.github/workflows/shellcheck.yml +1 -1
- data/vendor/liburing/.gitignore +6 -123
- data/vendor/liburing/CHANGELOG +35 -0
- data/vendor/liburing/CITATION.cff +11 -0
- data/vendor/liburing/LICENSE +16 -3
- data/vendor/liburing/Makefile +3 -1
- data/vendor/liburing/Makefile.common +1 -0
- data/vendor/liburing/README +14 -2
- data/vendor/liburing/SECURITY.md +6 -0
- data/vendor/liburing/configure +16 -15
- data/vendor/liburing/examples/Makefile +4 -1
- data/vendor/liburing/examples/io_uring-udp.c +395 -0
- data/vendor/liburing/examples/poll-bench.c +101 -0
- data/vendor/liburing/examples/send-zerocopy.c +339 -0
- data/vendor/liburing/liburing.spec +1 -1
- data/vendor/liburing/man/io_uring.7 +38 -11
- data/vendor/liburing/man/io_uring_buf_ring_add.3 +53 -0
- data/vendor/liburing/man/io_uring_buf_ring_advance.3 +31 -0
- data/vendor/liburing/man/io_uring_buf_ring_cq_advance.3 +41 -0
- data/vendor/liburing/man/io_uring_buf_ring_init.3 +30 -0
- data/vendor/liburing/man/io_uring_buf_ring_mask.3 +27 -0
- data/vendor/liburing/man/io_uring_cq_advance.3 +29 -15
- data/vendor/liburing/man/io_uring_cq_has_overflow.3 +25 -0
- data/vendor/liburing/man/io_uring_cq_ready.3 +9 -8
- data/vendor/liburing/man/io_uring_cqe_get_data.3 +32 -13
- data/vendor/liburing/man/io_uring_cqe_get_data64.3 +1 -0
- data/vendor/liburing/man/io_uring_cqe_seen.3 +22 -12
- data/vendor/liburing/man/io_uring_enter.2 +249 -32
- data/vendor/liburing/man/io_uring_enter2.2 +1 -0
- data/vendor/liburing/man/io_uring_free_probe.3 +11 -8
- data/vendor/liburing/man/io_uring_get_events.3 +33 -0
- data/vendor/liburing/man/io_uring_get_probe.3 +9 -8
- data/vendor/liburing/man/io_uring_get_sqe.3 +29 -10
- data/vendor/liburing/man/io_uring_opcode_supported.3 +11 -10
- data/vendor/liburing/man/io_uring_peek_cqe.3 +38 -0
- data/vendor/liburing/man/io_uring_prep_accept.3 +197 -0
- data/vendor/liburing/man/io_uring_prep_accept_direct.3 +1 -0
- data/vendor/liburing/man/io_uring_prep_cancel.3 +118 -0
- data/vendor/liburing/man/io_uring_prep_cancel64.3 +1 -0
- data/vendor/liburing/man/io_uring_prep_close.3 +59 -0
- data/vendor/liburing/man/io_uring_prep_close_direct.3 +1 -0
- data/vendor/liburing/man/io_uring_prep_connect.3 +66 -0
- data/vendor/liburing/man/io_uring_prep_fadvise.3 +59 -0
- data/vendor/liburing/man/io_uring_prep_fallocate.3 +59 -0
- data/vendor/liburing/man/io_uring_prep_files_update.3 +92 -0
- data/vendor/liburing/man/io_uring_prep_fsync.3 +70 -0
- data/vendor/liburing/man/io_uring_prep_link.3 +1 -0
- data/vendor/liburing/man/io_uring_prep_linkat.3 +91 -0
- data/vendor/liburing/man/io_uring_prep_madvise.3 +56 -0
- data/vendor/liburing/man/io_uring_prep_mkdir.3 +1 -0
- data/vendor/liburing/man/io_uring_prep_mkdirat.3 +83 -0
- data/vendor/liburing/man/io_uring_prep_msg_ring.3 +39 -25
- data/vendor/liburing/man/io_uring_prep_multishot_accept.3 +1 -0
- data/vendor/liburing/man/io_uring_prep_multishot_accept_direct.3 +1 -0
- data/vendor/liburing/man/io_uring_prep_nop.3 +28 -0
- data/vendor/liburing/man/io_uring_prep_openat.3 +117 -0
- data/vendor/liburing/man/io_uring_prep_openat2.3 +117 -0
- data/vendor/liburing/man/io_uring_prep_openat2_direct.3 +1 -0
- data/vendor/liburing/man/io_uring_prep_openat_direct.3 +1 -0
- data/vendor/liburing/man/io_uring_prep_poll_add.3 +72 -0
- data/vendor/liburing/man/io_uring_prep_poll_multishot.3 +1 -0
- data/vendor/liburing/man/io_uring_prep_poll_remove.3 +55 -0
- data/vendor/liburing/man/io_uring_prep_poll_update.3 +89 -0
- data/vendor/liburing/man/io_uring_prep_provide_buffers.3 +131 -0
- data/vendor/liburing/man/io_uring_prep_read.3 +33 -14
- data/vendor/liburing/man/io_uring_prep_read_fixed.3 +39 -21
- data/vendor/liburing/man/io_uring_prep_readv.3 +49 -15
- data/vendor/liburing/man/io_uring_prep_readv2.3 +49 -17
- data/vendor/liburing/man/io_uring_prep_recv.3 +105 -0
- data/vendor/liburing/man/io_uring_prep_recv_multishot.3 +1 -0
- data/vendor/liburing/man/io_uring_prep_recvmsg.3 +124 -0
- data/vendor/liburing/man/io_uring_prep_recvmsg_multishot.3 +1 -0
- data/vendor/liburing/man/io_uring_prep_remove_buffers.3 +52 -0
- data/vendor/liburing/man/io_uring_prep_rename.3 +1 -0
- data/vendor/liburing/man/io_uring_prep_renameat.3 +96 -0
- data/vendor/liburing/man/io_uring_prep_send.3 +57 -0
- data/vendor/liburing/man/io_uring_prep_send_zc.3 +64 -0
- data/vendor/liburing/man/io_uring_prep_sendmsg.3 +69 -0
- data/vendor/liburing/man/io_uring_prep_shutdown.3 +53 -0
- data/vendor/liburing/man/io_uring_prep_socket.3 +118 -0
- data/vendor/liburing/man/io_uring_prep_socket_direct.3 +1 -0
- data/vendor/liburing/man/io_uring_prep_socket_direct_alloc.3 +1 -0
- data/vendor/liburing/man/io_uring_prep_splice.3 +80 -0
- data/vendor/liburing/man/io_uring_prep_statx.3 +74 -0
- data/vendor/liburing/man/io_uring_prep_symlink.3 +1 -0
- data/vendor/liburing/man/io_uring_prep_symlinkat.3 +85 -0
- data/vendor/liburing/man/io_uring_prep_sync_file_range.3 +59 -0
- data/vendor/liburing/man/io_uring_prep_tee.3 +74 -0
- data/vendor/liburing/man/io_uring_prep_timeout.3 +95 -0
- data/vendor/liburing/man/io_uring_prep_timeout_remove.3 +1 -0
- data/vendor/liburing/man/io_uring_prep_timeout_update.3 +98 -0
- data/vendor/liburing/man/io_uring_prep_unlink.3 +1 -0
- data/vendor/liburing/man/io_uring_prep_unlinkat.3 +82 -0
- data/vendor/liburing/man/io_uring_prep_write.3 +32 -15
- data/vendor/liburing/man/io_uring_prep_write_fixed.3 +39 -21
- data/vendor/liburing/man/io_uring_prep_writev.3 +50 -16
- data/vendor/liburing/man/io_uring_prep_writev2.3 +50 -17
- data/vendor/liburing/man/io_uring_queue_exit.3 +3 -4
- data/vendor/liburing/man/io_uring_queue_init.3 +58 -13
- data/vendor/liburing/man/io_uring_queue_init_params.3 +1 -0
- data/vendor/liburing/man/io_uring_recvmsg_cmsg_firsthdr.3 +1 -0
- data/vendor/liburing/man/io_uring_recvmsg_cmsg_nexthdr.3 +1 -0
- data/vendor/liburing/man/io_uring_recvmsg_name.3 +1 -0
- data/vendor/liburing/man/io_uring_recvmsg_out.3 +78 -0
- data/vendor/liburing/man/io_uring_recvmsg_payload.3 +1 -0
- data/vendor/liburing/man/io_uring_recvmsg_payload_length.3 +1 -0
- data/vendor/liburing/man/io_uring_recvmsg_validate.3 +1 -0
- data/vendor/liburing/man/io_uring_register.2 +153 -13
- data/vendor/liburing/man/io_uring_register_buf_ring.3 +140 -0
- data/vendor/liburing/man/io_uring_register_buffers.3 +32 -12
- data/vendor/liburing/man/io_uring_register_eventfd.3 +51 -0
- data/vendor/liburing/man/io_uring_register_eventfd_async.3 +1 -0
- data/vendor/liburing/man/io_uring_register_file_alloc_range.3 +52 -0
- data/vendor/liburing/man/io_uring_register_files.3 +33 -11
- data/vendor/liburing/man/io_uring_register_files_sparse.3 +1 -0
- data/vendor/liburing/man/io_uring_register_iowq_aff.3 +61 -0
- data/vendor/liburing/man/io_uring_register_iowq_max_workers.3 +71 -0
- data/vendor/liburing/man/io_uring_register_ring_fd.3 +49 -0
- data/vendor/liburing/man/io_uring_register_sync_cancel.3 +71 -0
- data/vendor/liburing/man/io_uring_setup.2 +119 -13
- data/vendor/liburing/man/io_uring_sq_ready.3 +14 -8
- data/vendor/liburing/man/io_uring_sq_space_left.3 +9 -9
- data/vendor/liburing/man/io_uring_sqe_set_data.3 +29 -11
- data/vendor/liburing/man/io_uring_sqe_set_data64.3 +1 -0
- data/vendor/liburing/man/io_uring_sqe_set_flags.3 +38 -11
- data/vendor/liburing/man/io_uring_sqring_wait.3 +13 -9
- data/vendor/liburing/man/io_uring_submit.3 +29 -12
- data/vendor/liburing/man/io_uring_submit_and_get_events.3 +31 -0
- data/vendor/liburing/man/io_uring_submit_and_wait.3 +16 -12
- data/vendor/liburing/man/io_uring_submit_and_wait_timeout.3 +30 -23
- data/vendor/liburing/man/io_uring_unregister_buf_ring.3 +30 -0
- data/vendor/liburing/man/io_uring_unregister_buffers.3 +11 -10
- data/vendor/liburing/man/io_uring_unregister_eventfd.3 +1 -0
- data/vendor/liburing/man/io_uring_unregister_files.3 +11 -10
- data/vendor/liburing/man/io_uring_unregister_iowq_aff.3 +1 -0
- data/vendor/liburing/man/io_uring_unregister_ring_fd.3 +32 -0
- data/vendor/liburing/man/io_uring_wait_cqe.3 +19 -12
- data/vendor/liburing/man/io_uring_wait_cqe_nr.3 +21 -14
- data/vendor/liburing/man/io_uring_wait_cqe_timeout.3 +27 -13
- data/vendor/liburing/man/io_uring_wait_cqes.3 +24 -14
- data/vendor/liburing/src/Makefile +8 -7
- data/vendor/liburing/src/arch/aarch64/lib.h +48 -0
- data/vendor/liburing/src/arch/aarch64/syscall.h +0 -4
- data/vendor/liburing/src/arch/generic/lib.h +0 -4
- data/vendor/liburing/src/arch/generic/syscall.h +29 -16
- data/vendor/liburing/src/arch/syscall-defs.h +41 -14
- data/vendor/liburing/src/arch/x86/lib.h +0 -21
- data/vendor/liburing/src/arch/x86/syscall.h +146 -10
- data/vendor/liburing/src/include/liburing/io_uring.h +245 -5
- data/vendor/liburing/src/include/liburing.h +468 -35
- data/vendor/liburing/src/int_flags.h +1 -0
- data/vendor/liburing/src/lib.h +20 -16
- data/vendor/liburing/src/liburing.map +16 -0
- data/vendor/liburing/src/nolibc.c +1 -1
- data/vendor/liburing/src/queue.c +87 -55
- data/vendor/liburing/src/register.c +129 -53
- data/vendor/liburing/src/setup.c +65 -28
- data/vendor/liburing/src/syscall.c +14 -32
- data/vendor/liburing/src/syscall.h +12 -64
- data/vendor/liburing/test/{232c93d07b74-test.c → 232c93d07b74.c} +8 -9
- data/vendor/liburing/test/{35fa71a030ca-test.c → 35fa71a030ca.c} +4 -4
- data/vendor/liburing/test/{500f9fbadef8-test.c → 500f9fbadef8.c} +7 -7
- data/vendor/liburing/test/{7ad0e4b2f83c-test.c → 7ad0e4b2f83c.c} +8 -7
- data/vendor/liburing/test/{8a9973408177-test.c → 8a9973408177.c} +4 -3
- data/vendor/liburing/test/{917257daa0fe-test.c → 917257daa0fe.c} +3 -2
- data/vendor/liburing/test/Makefile +60 -62
- data/vendor/liburing/test/{a0908ae19763-test.c → a0908ae19763.c} +3 -2
- data/vendor/liburing/test/{a4c0b3decb33-test.c → a4c0b3decb33.c} +3 -2
- data/vendor/liburing/test/accept-link.c +5 -4
- data/vendor/liburing/test/accept-reuse.c +17 -16
- data/vendor/liburing/test/accept-test.c +14 -10
- data/vendor/liburing/test/accept.c +529 -107
- data/vendor/liburing/test/across-fork.c +7 -6
- data/vendor/liburing/test/{b19062a56726-test.c → b19062a56726.c} +3 -2
- data/vendor/liburing/test/{b5837bd5311d-test.c → b5837bd5311d.c} +10 -9
- data/vendor/liburing/test/buf-ring.c +420 -0
- data/vendor/liburing/test/{ce593a6c480a-test.c → ce593a6c480a.c} +15 -12
- data/vendor/liburing/test/connect.c +8 -7
- data/vendor/liburing/test/cq-full.c +5 -4
- data/vendor/liburing/test/cq-overflow.c +242 -12
- data/vendor/liburing/test/cq-peek-batch.c +5 -4
- data/vendor/liburing/test/cq-ready.c +5 -4
- data/vendor/liburing/test/cq-size.c +5 -4
- data/vendor/liburing/test/{d4ae271dfaae-test.c → d4ae271dfaae.c} +2 -2
- data/vendor/liburing/test/{d77a67ed5f27-test.c → d77a67ed5f27.c} +6 -6
- data/vendor/liburing/test/defer-taskrun.c +336 -0
- data/vendor/liburing/test/defer.c +26 -14
- data/vendor/liburing/test/double-poll-crash.c +15 -5
- data/vendor/liburing/test/drop-submit.c +5 -3
- data/vendor/liburing/test/{eeed8b54e0df-test.c → eeed8b54e0df.c} +7 -6
- data/vendor/liburing/test/empty-eownerdead.c +4 -4
- data/vendor/liburing/test/eventfd-disable.c +48 -20
- data/vendor/liburing/test/eventfd-reg.c +10 -9
- data/vendor/liburing/test/eventfd-ring.c +13 -12
- data/vendor/liburing/test/eventfd.c +13 -12
- data/vendor/liburing/test/exit-no-cleanup.c +1 -1
- data/vendor/liburing/test/fadvise.c +3 -3
- data/vendor/liburing/test/fallocate.c +16 -9
- data/vendor/liburing/test/{fc2a85cb02ef-test.c → fc2a85cb02ef.c} +4 -3
- data/vendor/liburing/test/fd-pass.c +187 -0
- data/vendor/liburing/test/file-register.c +302 -36
- data/vendor/liburing/test/file-update.c +62 -4
- data/vendor/liburing/test/file-verify.c +6 -2
- data/vendor/liburing/test/files-exit-hang-poll.c +11 -25
- data/vendor/liburing/test/files-exit-hang-timeout.c +13 -10
- data/vendor/liburing/test/fixed-buf-iter.c +115 -0
- data/vendor/liburing/test/fixed-link.c +10 -10
- data/vendor/liburing/test/fixed-reuse.c +160 -0
- data/vendor/liburing/test/fpos.c +6 -3
- data/vendor/liburing/test/fsync.c +3 -3
- data/vendor/liburing/test/hardlink.c +10 -6
- data/vendor/liburing/test/helpers.c +137 -4
- data/vendor/liburing/test/helpers.h +27 -0
- data/vendor/liburing/test/io-cancel.c +16 -11
- data/vendor/liburing/test/io_uring_enter.c +46 -81
- data/vendor/liburing/test/io_uring_passthrough.c +451 -0
- data/vendor/liburing/test/io_uring_register.c +59 -229
- data/vendor/liburing/test/io_uring_setup.c +24 -29
- data/vendor/liburing/test/iopoll-leak.c +85 -0
- data/vendor/liburing/test/iopoll.c +16 -9
- data/vendor/liburing/test/lfs-openat-write.c +3 -1
- data/vendor/liburing/test/link-timeout.c +4 -3
- data/vendor/liburing/test/link.c +8 -7
- data/vendor/liburing/test/madvise.c +2 -2
- data/vendor/liburing/test/mkdir.c +9 -5
- data/vendor/liburing/test/msg-ring.c +46 -20
- data/vendor/liburing/test/multicqes_drain.c +51 -12
- data/vendor/liburing/test/nolibc.c +60 -0
- data/vendor/liburing/test/nop.c +78 -16
- data/vendor/liburing/test/nvme.h +168 -0
- data/vendor/liburing/test/open-direct-link.c +188 -0
- data/vendor/liburing/test/open-direct-pick.c +180 -0
- data/vendor/liburing/test/openat2.c +3 -3
- data/vendor/liburing/test/poll-cancel-all.c +472 -0
- data/vendor/liburing/test/poll-link.c +9 -18
- data/vendor/liburing/test/poll-mshot-overflow.c +162 -0
- data/vendor/liburing/test/poll-mshot-update.c +83 -33
- data/vendor/liburing/test/pollfree.c +2 -2
- data/vendor/liburing/test/read-before-exit.c +112 -0
- data/vendor/liburing/test/read-write.c +83 -1
- data/vendor/liburing/test/recv-msgall-stream.c +398 -0
- data/vendor/liburing/test/recv-msgall.c +265 -0
- data/vendor/liburing/test/recv-multishot.c +505 -0
- data/vendor/liburing/test/rename.c +2 -5
- data/vendor/liburing/test/ring-leak.c +97 -0
- data/vendor/liburing/test/ringbuf-read.c +200 -0
- data/vendor/liburing/test/rsrc_tags.c +25 -13
- data/vendor/liburing/test/runtests-quiet.sh +11 -0
- data/vendor/liburing/test/runtests.sh +18 -20
- data/vendor/liburing/test/rw_merge_test.c +3 -2
- data/vendor/liburing/test/send-zerocopy.c +684 -0
- data/vendor/liburing/test/send_recv.c +49 -2
- data/vendor/liburing/test/send_recvmsg.c +165 -55
- data/vendor/liburing/test/shutdown.c +3 -4
- data/vendor/liburing/test/sigfd-deadlock.c +22 -8
- data/vendor/liburing/test/single-issuer.c +171 -0
- data/vendor/liburing/test/socket-rw-eagain.c +2 -12
- data/vendor/liburing/test/socket-rw-offset.c +2 -11
- data/vendor/liburing/test/socket-rw.c +2 -11
- data/vendor/liburing/test/socket.c +409 -0
- data/vendor/liburing/test/sq-poll-dup.c +1 -1
- data/vendor/liburing/test/sq-poll-share.c +1 -1
- data/vendor/liburing/test/statx.c +2 -2
- data/vendor/liburing/test/submit-and-wait.c +108 -0
- data/vendor/liburing/test/submit-link-fail.c +5 -3
- data/vendor/liburing/test/submit-reuse.c +0 -2
- data/vendor/liburing/test/sync-cancel.c +235 -0
- data/vendor/liburing/test/test.h +35 -0
- data/vendor/liburing/test/timeout-overflow.c +11 -11
- data/vendor/liburing/test/timeout.c +7 -7
- data/vendor/liburing/test/tty-write-dpoll.c +60 -0
- data/vendor/liburing/test/unlink.c +1 -1
- data/vendor/liburing/test/xattr.c +425 -0
- metadata +160 -52
- data/Gemfile.lock +0 -78
@@ -6,6 +6,10 @@
|
|
6
6
|
#define _XOPEN_SOURCE 500 /* Required for glibc to expose sigset_t */
|
7
7
|
#endif
|
8
8
|
|
9
|
+
#ifndef _GNU_SOURCE
|
10
|
+
#define _GNU_SOURCE /* Required for musl to expose cpu_set_t */
|
11
|
+
#endif
|
12
|
+
|
9
13
|
#include <sys/socket.h>
|
10
14
|
#include <sys/stat.h>
|
11
15
|
#include <sys/uio.h>
|
@@ -14,6 +18,7 @@
|
|
14
18
|
#include <stdbool.h>
|
15
19
|
#include <inttypes.h>
|
16
20
|
#include <time.h>
|
21
|
+
#include <fcntl.h>
|
17
22
|
#include <sched.h>
|
18
23
|
#include <linux/swab.h>
|
19
24
|
#include "liburing/compat.h"
|
@@ -21,11 +26,47 @@
|
|
21
26
|
#include "liburing/barrier.h"
|
22
27
|
|
23
28
|
#ifndef uring_unlikely
|
24
|
-
#
|
29
|
+
#define uring_unlikely(cond) __builtin_expect(!!(cond), 0)
|
25
30
|
#endif
|
26
31
|
|
27
32
|
#ifndef uring_likely
|
28
|
-
#
|
33
|
+
#define uring_likely(cond) __builtin_expect(!!(cond), 1)
|
34
|
+
#endif
|
35
|
+
|
36
|
+
#ifdef __alpha__
|
37
|
+
/*
|
38
|
+
* alpha and mips are the exceptions, all other architectures have
|
39
|
+
* common numbers for new system calls.
|
40
|
+
*/
|
41
|
+
#ifndef __NR_io_uring_setup
|
42
|
+
#define __NR_io_uring_setup 535
|
43
|
+
#endif
|
44
|
+
#ifndef __NR_io_uring_enter
|
45
|
+
#define __NR_io_uring_enter 536
|
46
|
+
#endif
|
47
|
+
#ifndef __NR_io_uring_register
|
48
|
+
#define __NR_io_uring_register 537
|
49
|
+
#endif
|
50
|
+
#elif defined __mips__
|
51
|
+
#ifndef __NR_io_uring_setup
|
52
|
+
#define __NR_io_uring_setup (__NR_Linux + 425)
|
53
|
+
#endif
|
54
|
+
#ifndef __NR_io_uring_enter
|
55
|
+
#define __NR_io_uring_enter (__NR_Linux + 426)
|
56
|
+
#endif
|
57
|
+
#ifndef __NR_io_uring_register
|
58
|
+
#define __NR_io_uring_register (__NR_Linux + 427)
|
59
|
+
#endif
|
60
|
+
#else /* !__alpha__ and !__mips__ */
|
61
|
+
#ifndef __NR_io_uring_setup
|
62
|
+
#define __NR_io_uring_setup 425
|
63
|
+
#endif
|
64
|
+
#ifndef __NR_io_uring_enter
|
65
|
+
#define __NR_io_uring_enter 426
|
66
|
+
#endif
|
67
|
+
#ifndef __NR_io_uring_register
|
68
|
+
#define __NR_io_uring_register 427
|
69
|
+
#endif
|
29
70
|
#endif
|
30
71
|
|
31
72
|
#ifdef __cplusplus
|
@@ -38,7 +79,9 @@ extern "C" {
|
|
38
79
|
struct io_uring_sq {
|
39
80
|
unsigned *khead;
|
40
81
|
unsigned *ktail;
|
82
|
+
// Deprecated: use `ring_mask` instead of `*kring_mask`
|
41
83
|
unsigned *kring_mask;
|
84
|
+
// Deprecated: use `ring_entries` instead of `*kring_entries`
|
42
85
|
unsigned *kring_entries;
|
43
86
|
unsigned *kflags;
|
44
87
|
unsigned *kdropped;
|
@@ -51,13 +94,18 @@ struct io_uring_sq {
|
|
51
94
|
size_t ring_sz;
|
52
95
|
void *ring_ptr;
|
53
96
|
|
54
|
-
unsigned
|
97
|
+
unsigned ring_mask;
|
98
|
+
unsigned ring_entries;
|
99
|
+
|
100
|
+
unsigned pad[2];
|
55
101
|
};
|
56
102
|
|
57
103
|
struct io_uring_cq {
|
58
104
|
unsigned *khead;
|
59
105
|
unsigned *ktail;
|
106
|
+
// Deprecated: use `ring_mask` instead of `*kring_mask`
|
60
107
|
unsigned *kring_mask;
|
108
|
+
// Deprecated: use `ring_entries` instead of `*kring_entries`
|
61
109
|
unsigned *kring_entries;
|
62
110
|
unsigned *kflags;
|
63
111
|
unsigned *koverflow;
|
@@ -66,7 +114,10 @@ struct io_uring_cq {
|
|
66
114
|
size_t ring_sz;
|
67
115
|
void *ring_ptr;
|
68
116
|
|
69
|
-
unsigned
|
117
|
+
unsigned ring_mask;
|
118
|
+
unsigned ring_entries;
|
119
|
+
|
120
|
+
unsigned pad[2];
|
70
121
|
};
|
71
122
|
|
72
123
|
struct io_uring {
|
@@ -100,7 +151,8 @@ struct io_uring_probe *io_uring_get_probe(void);
|
|
100
151
|
*/
|
101
152
|
void io_uring_free_probe(struct io_uring_probe *probe);
|
102
153
|
|
103
|
-
static inline int io_uring_opcode_supported(const struct io_uring_probe *p,
|
154
|
+
static inline int io_uring_opcode_supported(const struct io_uring_probe *p,
|
155
|
+
int op)
|
104
156
|
{
|
105
157
|
if (op > p->last_op)
|
106
158
|
return 0;
|
@@ -136,6 +188,7 @@ int io_uring_register_buffers(struct io_uring *ring, const struct iovec *iovecs,
|
|
136
188
|
int io_uring_register_buffers_tags(struct io_uring *ring,
|
137
189
|
const struct iovec *iovecs,
|
138
190
|
const __u64 *tags, unsigned nr);
|
191
|
+
int io_uring_register_buffers_sparse(struct io_uring *ring, unsigned nr);
|
139
192
|
int io_uring_register_buffers_update_tag(struct io_uring *ring,
|
140
193
|
unsigned off,
|
141
194
|
const struct iovec *iovecs,
|
@@ -146,13 +199,14 @@ int io_uring_register_files(struct io_uring *ring, const int *files,
|
|
146
199
|
unsigned nr_files);
|
147
200
|
int io_uring_register_files_tags(struct io_uring *ring, const int *files,
|
148
201
|
const __u64 *tags, unsigned nr);
|
202
|
+
int io_uring_register_files_sparse(struct io_uring *ring, unsigned nr);
|
149
203
|
int io_uring_register_files_update_tag(struct io_uring *ring, unsigned off,
|
150
204
|
const int *files, const __u64 *tags,
|
151
205
|
unsigned nr_files);
|
152
206
|
|
153
207
|
int io_uring_unregister_files(struct io_uring *ring);
|
154
208
|
int io_uring_register_files_update(struct io_uring *ring, unsigned off,
|
155
|
-
int *files, unsigned nr_files);
|
209
|
+
const int *files, unsigned nr_files);
|
156
210
|
int io_uring_register_eventfd(struct io_uring *ring, int fd);
|
157
211
|
int io_uring_register_eventfd_async(struct io_uring *ring, int fd);
|
158
212
|
int io_uring_unregister_eventfd(struct io_uring *ring);
|
@@ -172,6 +226,29 @@ int io_uring_register_iowq_max_workers(struct io_uring *ring,
|
|
172
226
|
unsigned int *values);
|
173
227
|
int io_uring_register_ring_fd(struct io_uring *ring);
|
174
228
|
int io_uring_unregister_ring_fd(struct io_uring *ring);
|
229
|
+
int io_uring_register_buf_ring(struct io_uring *ring,
|
230
|
+
struct io_uring_buf_reg *reg, unsigned int flags);
|
231
|
+
int io_uring_unregister_buf_ring(struct io_uring *ring, int bgid);
|
232
|
+
int io_uring_register_sync_cancel(struct io_uring *ring,
|
233
|
+
struct io_uring_sync_cancel_reg *reg);
|
234
|
+
|
235
|
+
int io_uring_register_file_alloc_range(struct io_uring *ring,
|
236
|
+
unsigned off, unsigned len);
|
237
|
+
|
238
|
+
int io_uring_get_events(struct io_uring *ring);
|
239
|
+
int io_uring_submit_and_get_events(struct io_uring *ring);
|
240
|
+
|
241
|
+
/*
|
242
|
+
* io_uring syscalls.
|
243
|
+
*/
|
244
|
+
int io_uring_enter(unsigned int fd, unsigned int to_submit,
|
245
|
+
unsigned int min_complete, unsigned int flags, sigset_t *sig);
|
246
|
+
int io_uring_enter2(unsigned int fd, unsigned int to_submit,
|
247
|
+
unsigned int min_complete, unsigned int flags,
|
248
|
+
sigset_t *sig, size_t sz);
|
249
|
+
int io_uring_setup(unsigned int entries, struct io_uring_params *p);
|
250
|
+
int io_uring_register(unsigned int fd, unsigned int opcode, const void *arg,
|
251
|
+
unsigned int nr_args);
|
175
252
|
|
176
253
|
/*
|
177
254
|
* Helper for the peek/wait single cqe functions. Exported because of that,
|
@@ -183,6 +260,16 @@ int __io_uring_get_cqe(struct io_uring *ring,
|
|
183
260
|
|
184
261
|
#define LIBURING_UDATA_TIMEOUT ((__u64) -1)
|
185
262
|
|
263
|
+
/*
|
264
|
+
* Calculates the step size for CQE iteration.
|
265
|
+
* For standard CQE's its 1, for big CQE's its two.
|
266
|
+
*/
|
267
|
+
#define io_uring_cqe_shift(ring) \
|
268
|
+
(!!((ring)->flags & IORING_SETUP_CQE32))
|
269
|
+
|
270
|
+
#define io_uring_cqe_index(ring,ptr,mask) \
|
271
|
+
(((ptr) & (mask)) << io_uring_cqe_shift(ring))
|
272
|
+
|
186
273
|
#define io_uring_for_each_cqe(ring, head, cqe) \
|
187
274
|
/* \
|
188
275
|
* io_uring_smp_load_acquire() enforces the order of tail \
|
@@ -190,7 +277,7 @@ int __io_uring_get_cqe(struct io_uring *ring,
|
|
190
277
|
*/ \
|
191
278
|
for (head = *(ring)->cq.khead; \
|
192
279
|
(cqe = (head != io_uring_smp_load_acquire((ring)->cq.ktail) ? \
|
193
|
-
&(ring)->cq.cqes[head
|
280
|
+
&(ring)->cq.cqes[io_uring_cqe_index(ring, head, (ring)->cq.ring_mask)] : NULL)); \
|
194
281
|
head++) \
|
195
282
|
|
196
283
|
/*
|
@@ -288,7 +375,8 @@ static inline void io_uring_prep_rw(int op, struct io_uring_sqe *sqe, int fd,
|
|
288
375
|
sqe->buf_index = 0;
|
289
376
|
sqe->personality = 0;
|
290
377
|
sqe->file_index = 0;
|
291
|
-
sqe->
|
378
|
+
sqe->addr3 = 0;
|
379
|
+
sqe->__pad2[0] = 0;
|
292
380
|
}
|
293
381
|
|
294
382
|
/**
|
@@ -346,7 +434,8 @@ static inline void io_uring_prep_readv(struct io_uring_sqe *sqe, int fd,
|
|
346
434
|
|
347
435
|
static inline void io_uring_prep_readv2(struct io_uring_sqe *sqe, int fd,
|
348
436
|
const struct iovec *iovecs,
|
349
|
-
unsigned nr_vecs, __u64 offset,
|
437
|
+
unsigned nr_vecs, __u64 offset,
|
438
|
+
int flags)
|
350
439
|
{
|
351
440
|
io_uring_prep_readv(sqe, fd, iovecs, nr_vecs, offset);
|
352
441
|
sqe->rw_flags = flags;
|
@@ -369,7 +458,8 @@ static inline void io_uring_prep_writev(struct io_uring_sqe *sqe, int fd,
|
|
369
458
|
|
370
459
|
static inline void io_uring_prep_writev2(struct io_uring_sqe *sqe, int fd,
|
371
460
|
const struct iovec *iovecs,
|
372
|
-
unsigned nr_vecs, __u64 offset,
|
461
|
+
unsigned nr_vecs, __u64 offset,
|
462
|
+
int flags)
|
373
463
|
{
|
374
464
|
io_uring_prep_writev(sqe, fd, iovecs, nr_vecs, offset);
|
375
465
|
sqe->rw_flags = flags;
|
@@ -390,8 +480,16 @@ static inline void io_uring_prep_recvmsg(struct io_uring_sqe *sqe, int fd,
|
|
390
480
|
sqe->msg_flags = flags;
|
391
481
|
}
|
392
482
|
|
483
|
+
static inline void io_uring_prep_recvmsg_multishot(struct io_uring_sqe *sqe, int fd,
|
484
|
+
struct msghdr *msg, unsigned flags)
|
485
|
+
{
|
486
|
+
io_uring_prep_recvmsg(sqe, fd, msg, flags);
|
487
|
+
sqe->ioprio |= IORING_RECV_MULTISHOT;
|
488
|
+
}
|
489
|
+
|
393
490
|
static inline void io_uring_prep_sendmsg(struct io_uring_sqe *sqe, int fd,
|
394
|
-
const struct msghdr *msg,
|
491
|
+
const struct msghdr *msg,
|
492
|
+
unsigned flags)
|
395
493
|
{
|
396
494
|
io_uring_prep_rw(IORING_OP_SENDMSG, sqe, fd, msg, 1, 0);
|
397
495
|
sqe->msg_flags = flags;
|
@@ -494,14 +592,46 @@ static inline void io_uring_prep_accept_direct(struct io_uring_sqe *sqe, int fd,
|
|
494
592
|
__io_uring_set_target_fixed_file(sqe, file_index);
|
495
593
|
}
|
496
594
|
|
497
|
-
static inline void
|
498
|
-
|
595
|
+
static inline void io_uring_prep_multishot_accept(struct io_uring_sqe *sqe,
|
596
|
+
int fd, struct sockaddr *addr,
|
597
|
+
socklen_t *addrlen, int flags)
|
598
|
+
{
|
599
|
+
io_uring_prep_accept(sqe, fd, addr, addrlen, flags);
|
600
|
+
sqe->ioprio |= IORING_ACCEPT_MULTISHOT;
|
601
|
+
}
|
602
|
+
|
603
|
+
/* multishot accept directly into the fixed file table */
|
604
|
+
static inline void io_uring_prep_multishot_accept_direct(struct io_uring_sqe *sqe,
|
605
|
+
int fd,
|
606
|
+
struct sockaddr *addr,
|
607
|
+
socklen_t *addrlen,
|
608
|
+
int flags)
|
609
|
+
{
|
610
|
+
io_uring_prep_multishot_accept(sqe, fd, addr, addrlen, flags);
|
611
|
+
__io_uring_set_target_fixed_file(sqe, IORING_FILE_INDEX_ALLOC - 1);
|
612
|
+
}
|
613
|
+
|
614
|
+
static inline void io_uring_prep_cancel64(struct io_uring_sqe *sqe,
|
615
|
+
__u64 user_data, int flags)
|
499
616
|
{
|
500
617
|
io_uring_prep_rw(IORING_OP_ASYNC_CANCEL, sqe, -1, NULL, 0, 0);
|
501
618
|
sqe->addr = user_data;
|
502
619
|
sqe->cancel_flags = (__u32) flags;
|
503
620
|
}
|
504
621
|
|
622
|
+
static inline void io_uring_prep_cancel(struct io_uring_sqe *sqe,
|
623
|
+
void *user_data, int flags)
|
624
|
+
{
|
625
|
+
io_uring_prep_cancel64(sqe, (__u64) (uintptr_t) user_data, flags);
|
626
|
+
}
|
627
|
+
|
628
|
+
static inline void io_uring_prep_cancel_fd(struct io_uring_sqe *sqe, int fd,
|
629
|
+
unsigned int flags)
|
630
|
+
{
|
631
|
+
io_uring_prep_rw(IORING_OP_ASYNC_CANCEL, sqe, fd, NULL, 0, 0);
|
632
|
+
sqe->cancel_flags = (__u32) flags | IORING_ASYNC_CANCEL_FD;
|
633
|
+
}
|
634
|
+
|
505
635
|
static inline void io_uring_prep_link_timeout(struct io_uring_sqe *sqe,
|
506
636
|
struct __kernel_timespec *ts,
|
507
637
|
unsigned flags)
|
@@ -528,14 +658,14 @@ static inline void io_uring_prep_files_update(struct io_uring_sqe *sqe,
|
|
528
658
|
static inline void io_uring_prep_fallocate(struct io_uring_sqe *sqe, int fd,
|
529
659
|
int mode, off_t offset, off_t len)
|
530
660
|
{
|
531
|
-
|
532
661
|
io_uring_prep_rw(IORING_OP_FALLOCATE, sqe, fd,
|
533
|
-
(
|
534
|
-
|
662
|
+
0, (unsigned int) mode, (__u64) offset);
|
663
|
+
sqe->addr = (__u64) len;
|
535
664
|
}
|
536
665
|
|
537
666
|
static inline void io_uring_prep_openat(struct io_uring_sqe *sqe, int dfd,
|
538
|
-
const char *path, int flags,
|
667
|
+
const char *path, int flags,
|
668
|
+
mode_t mode)
|
539
669
|
{
|
540
670
|
io_uring_prep_rw(IORING_OP_OPENAT, sqe, dfd, path, mode, 0);
|
541
671
|
sqe->open_flags = (__u32) flags;
|
@@ -551,7 +681,6 @@ static inline void io_uring_prep_openat_direct(struct io_uring_sqe *sqe,
|
|
551
681
|
__io_uring_set_target_fixed_file(sqe, file_index);
|
552
682
|
}
|
553
683
|
|
554
|
-
|
555
684
|
static inline void io_uring_prep_close(struct io_uring_sqe *sqe, int fd)
|
556
685
|
{
|
557
686
|
io_uring_prep_rw(IORING_OP_CLOSE, sqe, fd, NULL, 0, 0);
|
@@ -571,7 +700,8 @@ static inline void io_uring_prep_read(struct io_uring_sqe *sqe, int fd,
|
|
571
700
|
}
|
572
701
|
|
573
702
|
static inline void io_uring_prep_write(struct io_uring_sqe *sqe, int fd,
|
574
|
-
const void *buf, unsigned nbytes,
|
703
|
+
const void *buf, unsigned nbytes,
|
704
|
+
__u64 offset)
|
575
705
|
{
|
576
706
|
io_uring_prep_rw(IORING_OP_WRITE, sqe, fd, buf, nbytes, offset);
|
577
707
|
}
|
@@ -607,6 +737,42 @@ static inline void io_uring_prep_send(struct io_uring_sqe *sqe, int sockfd,
|
|
607
737
|
sqe->msg_flags = (__u32) flags;
|
608
738
|
}
|
609
739
|
|
740
|
+
static inline void io_uring_prep_send_zc(struct io_uring_sqe *sqe, int sockfd,
|
741
|
+
const void *buf, size_t len, int flags,
|
742
|
+
unsigned zc_flags)
|
743
|
+
{
|
744
|
+
io_uring_prep_rw(IORING_OP_SEND_ZC, sqe, sockfd, buf, (__u32) len, 0);
|
745
|
+
sqe->msg_flags = (__u32) flags;
|
746
|
+
sqe->ioprio = zc_flags;
|
747
|
+
}
|
748
|
+
|
749
|
+
static inline void io_uring_prep_send_zc_fixed(struct io_uring_sqe *sqe,
|
750
|
+
int sockfd, const void *buf,
|
751
|
+
size_t len, int flags,
|
752
|
+
unsigned zc_flags,
|
753
|
+
unsigned buf_index)
|
754
|
+
{
|
755
|
+
io_uring_prep_send_zc(sqe, sockfd, buf, len, flags, zc_flags);
|
756
|
+
sqe->ioprio |= IORING_RECVSEND_FIXED_BUF;
|
757
|
+
sqe->buf_index = buf_index;
|
758
|
+
}
|
759
|
+
|
760
|
+
static inline void io_uring_prep_sendmsg_zc(struct io_uring_sqe *sqe, int fd,
|
761
|
+
const struct msghdr *msg,
|
762
|
+
unsigned flags)
|
763
|
+
{
|
764
|
+
io_uring_prep_sendmsg(sqe, fd, msg, flags);
|
765
|
+
sqe->opcode = IORING_OP_SENDMSG_ZC;
|
766
|
+
}
|
767
|
+
|
768
|
+
static inline void io_uring_prep_send_set_addr(struct io_uring_sqe *sqe,
|
769
|
+
const struct sockaddr *dest_addr,
|
770
|
+
__u16 addr_len)
|
771
|
+
{
|
772
|
+
sqe->addr2 = (unsigned long)(const void *)dest_addr;
|
773
|
+
sqe->addr_len = addr_len;
|
774
|
+
}
|
775
|
+
|
610
776
|
static inline void io_uring_prep_recv(struct io_uring_sqe *sqe, int sockfd,
|
611
777
|
void *buf, size_t len, int flags)
|
612
778
|
{
|
@@ -614,6 +780,79 @@ static inline void io_uring_prep_recv(struct io_uring_sqe *sqe, int sockfd,
|
|
614
780
|
sqe->msg_flags = (__u32) flags;
|
615
781
|
}
|
616
782
|
|
783
|
+
static inline void io_uring_prep_recv_multishot(struct io_uring_sqe *sqe,
|
784
|
+
int sockfd, void *buf,
|
785
|
+
size_t len, int flags)
|
786
|
+
{
|
787
|
+
io_uring_prep_recv(sqe, sockfd, buf, len, flags);
|
788
|
+
sqe->ioprio |= IORING_RECV_MULTISHOT;
|
789
|
+
}
|
790
|
+
|
791
|
+
static inline struct io_uring_recvmsg_out *
|
792
|
+
io_uring_recvmsg_validate(void *buf, int buf_len, struct msghdr *msgh)
|
793
|
+
{
|
794
|
+
unsigned long header = msgh->msg_controllen + msgh->msg_namelen +
|
795
|
+
sizeof(struct io_uring_recvmsg_out);
|
796
|
+
if (buf_len < 0 || (unsigned long)buf_len < header)
|
797
|
+
return NULL;
|
798
|
+
return (struct io_uring_recvmsg_out *)buf;
|
799
|
+
}
|
800
|
+
|
801
|
+
static inline void *io_uring_recvmsg_name(struct io_uring_recvmsg_out *o)
|
802
|
+
{
|
803
|
+
return (void *) &o[1];
|
804
|
+
}
|
805
|
+
|
806
|
+
static inline struct cmsghdr *
|
807
|
+
io_uring_recvmsg_cmsg_firsthdr(struct io_uring_recvmsg_out *o,
|
808
|
+
struct msghdr *msgh)
|
809
|
+
{
|
810
|
+
if (o->controllen < sizeof(struct cmsghdr))
|
811
|
+
return NULL;
|
812
|
+
|
813
|
+
return (struct cmsghdr *)((unsigned char *) io_uring_recvmsg_name(o) +
|
814
|
+
msgh->msg_namelen);
|
815
|
+
}
|
816
|
+
|
817
|
+
static inline struct cmsghdr *
|
818
|
+
io_uring_recvmsg_cmsg_nexthdr(struct io_uring_recvmsg_out *o, struct msghdr *msgh,
|
819
|
+
struct cmsghdr *cmsg)
|
820
|
+
{
|
821
|
+
unsigned char *end;
|
822
|
+
|
823
|
+
if (cmsg->cmsg_len < sizeof(struct cmsghdr))
|
824
|
+
return NULL;
|
825
|
+
end = (unsigned char *) io_uring_recvmsg_cmsg_firsthdr(o, msgh) +
|
826
|
+
o->controllen;
|
827
|
+
cmsg = (struct cmsghdr *)((unsigned char *) cmsg +
|
828
|
+
CMSG_ALIGN(cmsg->cmsg_len));
|
829
|
+
|
830
|
+
if ((unsigned char *) (cmsg + 1) > end)
|
831
|
+
return NULL;
|
832
|
+
if (((unsigned char *) cmsg) + CMSG_ALIGN(cmsg->cmsg_len) > end)
|
833
|
+
return NULL;
|
834
|
+
|
835
|
+
return cmsg;
|
836
|
+
}
|
837
|
+
|
838
|
+
static inline void *io_uring_recvmsg_payload(struct io_uring_recvmsg_out *o,
|
839
|
+
struct msghdr *msgh)
|
840
|
+
{
|
841
|
+
return (void *)((unsigned char *)io_uring_recvmsg_name(o) +
|
842
|
+
msgh->msg_namelen + msgh->msg_controllen);
|
843
|
+
}
|
844
|
+
|
845
|
+
static inline unsigned int
|
846
|
+
io_uring_recvmsg_payload_length(struct io_uring_recvmsg_out *o,
|
847
|
+
int buf_len, struct msghdr *msgh)
|
848
|
+
{
|
849
|
+
unsigned long payload_start, payload_end;
|
850
|
+
|
851
|
+
payload_start = (unsigned long) io_uring_recvmsg_payload(o, msgh);
|
852
|
+
payload_end = (unsigned long) o + buf_len;
|
853
|
+
return (unsigned int) (payload_end - payload_start);
|
854
|
+
}
|
855
|
+
|
617
856
|
static inline void io_uring_prep_openat2(struct io_uring_sqe *sqe, int dfd,
|
618
857
|
const char *path, struct open_how *how)
|
619
858
|
{
|
@@ -669,15 +908,28 @@ static inline void io_uring_prep_unlinkat(struct io_uring_sqe *sqe, int dfd,
|
|
669
908
|
sqe->unlink_flags = (__u32) flags;
|
670
909
|
}
|
671
910
|
|
911
|
+
static inline void io_uring_prep_unlink(struct io_uring_sqe *sqe,
|
912
|
+
const char *path, int flags)
|
913
|
+
{
|
914
|
+
io_uring_prep_unlinkat(sqe, AT_FDCWD, path, flags);
|
915
|
+
}
|
916
|
+
|
672
917
|
static inline void io_uring_prep_renameat(struct io_uring_sqe *sqe, int olddfd,
|
673
918
|
const char *oldpath, int newdfd,
|
674
|
-
const char *newpath, int flags)
|
919
|
+
const char *newpath, unsigned int flags)
|
675
920
|
{
|
676
|
-
io_uring_prep_rw(IORING_OP_RENAMEAT, sqe, olddfd, oldpath,
|
921
|
+
io_uring_prep_rw(IORING_OP_RENAMEAT, sqe, olddfd, oldpath,
|
922
|
+
(__u32) newdfd,
|
677
923
|
(uint64_t) (uintptr_t) newpath);
|
678
924
|
sqe->rename_flags = (__u32) flags;
|
679
925
|
}
|
680
926
|
|
927
|
+
static inline void io_uring_prep_rename(struct io_uring_sqe *sqe,
|
928
|
+
const char *oldpath, const char *newpath)
|
929
|
+
{
|
930
|
+
io_uring_prep_renameat(sqe, AT_FDCWD, oldpath, AT_FDCWD, newpath, 0);
|
931
|
+
}
|
932
|
+
|
681
933
|
static inline void io_uring_prep_sync_file_range(struct io_uring_sqe *sqe,
|
682
934
|
int fd, unsigned len,
|
683
935
|
__u64 offset, int flags)
|
@@ -692,13 +944,26 @@ static inline void io_uring_prep_mkdirat(struct io_uring_sqe *sqe, int dfd,
|
|
692
944
|
io_uring_prep_rw(IORING_OP_MKDIRAT, sqe, dfd, path, mode, 0);
|
693
945
|
}
|
694
946
|
|
947
|
+
static inline void io_uring_prep_mkdir(struct io_uring_sqe *sqe,
|
948
|
+
const char *path, mode_t mode)
|
949
|
+
{
|
950
|
+
io_uring_prep_mkdirat(sqe, AT_FDCWD, path, mode);
|
951
|
+
}
|
952
|
+
|
695
953
|
static inline void io_uring_prep_symlinkat(struct io_uring_sqe *sqe,
|
696
|
-
|
954
|
+
const char *target, int newdirfd,
|
955
|
+
const char *linkpath)
|
697
956
|
{
|
698
957
|
io_uring_prep_rw(IORING_OP_SYMLINKAT, sqe, newdirfd, target, 0,
|
699
958
|
(uint64_t) (uintptr_t) linkpath);
|
700
959
|
}
|
701
960
|
|
961
|
+
static inline void io_uring_prep_symlink(struct io_uring_sqe *sqe,
|
962
|
+
const char *target, const char *linkpath)
|
963
|
+
{
|
964
|
+
io_uring_prep_symlinkat(sqe, target, AT_FDCWD, linkpath);
|
965
|
+
}
|
966
|
+
|
702
967
|
static inline void io_uring_prep_linkat(struct io_uring_sqe *sqe, int olddfd,
|
703
968
|
const char *oldpath, int newdfd,
|
704
969
|
const char *newpath, int flags)
|
@@ -708,6 +973,12 @@ static inline void io_uring_prep_linkat(struct io_uring_sqe *sqe, int olddfd,
|
|
708
973
|
sqe->hardlink_flags = (__u32) flags;
|
709
974
|
}
|
710
975
|
|
976
|
+
static inline void io_uring_prep_link(struct io_uring_sqe *sqe,
|
977
|
+
const char *oldpath, const char *newpath, int flags)
|
978
|
+
{
|
979
|
+
io_uring_prep_linkat(sqe, AT_FDCWD, oldpath, AT_FDCWD, newpath, flags);
|
980
|
+
}
|
981
|
+
|
711
982
|
static inline void io_uring_prep_msg_ring(struct io_uring_sqe *sqe, int fd,
|
712
983
|
unsigned int len, __u64 data,
|
713
984
|
unsigned int flags)
|
@@ -716,22 +987,100 @@ static inline void io_uring_prep_msg_ring(struct io_uring_sqe *sqe, int fd,
|
|
716
987
|
sqe->rw_flags = flags;
|
717
988
|
}
|
718
989
|
|
990
|
+
static inline void io_uring_prep_getxattr(struct io_uring_sqe *sqe,
|
991
|
+
const char *name,
|
992
|
+
char *value,
|
993
|
+
const char *path,
|
994
|
+
unsigned int len)
|
995
|
+
{
|
996
|
+
io_uring_prep_rw(IORING_OP_GETXATTR, sqe, 0, name, len,
|
997
|
+
(__u64) (uintptr_t) value);
|
998
|
+
sqe->addr3 = (__u64) (uintptr_t) path;
|
999
|
+
sqe->xattr_flags = 0;
|
1000
|
+
}
|
1001
|
+
|
1002
|
+
static inline void io_uring_prep_setxattr(struct io_uring_sqe *sqe,
|
1003
|
+
const char *name,
|
1004
|
+
const char *value,
|
1005
|
+
const char *path,
|
1006
|
+
int flags,
|
1007
|
+
unsigned int len)
|
1008
|
+
{
|
1009
|
+
io_uring_prep_rw(IORING_OP_SETXATTR, sqe, 0, name, len,
|
1010
|
+
(__u64) (uintptr_t) value);
|
1011
|
+
sqe->addr3 = (__u64) (uintptr_t) path;
|
1012
|
+
sqe->xattr_flags = flags;
|
1013
|
+
}
|
1014
|
+
|
1015
|
+
static inline void io_uring_prep_fgetxattr(struct io_uring_sqe *sqe,
|
1016
|
+
int fd,
|
1017
|
+
const char *name,
|
1018
|
+
char *value,
|
1019
|
+
unsigned int len)
|
1020
|
+
{
|
1021
|
+
io_uring_prep_rw(IORING_OP_FGETXATTR, sqe, fd, name, len,
|
1022
|
+
(__u64) (uintptr_t) value);
|
1023
|
+
sqe->xattr_flags = 0;
|
1024
|
+
}
|
1025
|
+
|
1026
|
+
static inline void io_uring_prep_fsetxattr(struct io_uring_sqe *sqe,
|
1027
|
+
int fd,
|
1028
|
+
const char *name,
|
1029
|
+
const char *value,
|
1030
|
+
int flags,
|
1031
|
+
unsigned int len)
|
1032
|
+
{
|
1033
|
+
io_uring_prep_rw(IORING_OP_FSETXATTR, sqe, fd, name, len,
|
1034
|
+
(__u64) (uintptr_t) value);
|
1035
|
+
sqe->xattr_flags = flags;
|
1036
|
+
}
|
1037
|
+
|
1038
|
+
static inline void io_uring_prep_socket(struct io_uring_sqe *sqe, int domain,
|
1039
|
+
int type, int protocol,
|
1040
|
+
unsigned int flags)
|
1041
|
+
{
|
1042
|
+
io_uring_prep_rw(IORING_OP_SOCKET, sqe, domain, NULL, protocol, type);
|
1043
|
+
sqe->rw_flags = flags;
|
1044
|
+
}
|
1045
|
+
|
1046
|
+
static inline void io_uring_prep_socket_direct(struct io_uring_sqe *sqe,
|
1047
|
+
int domain, int type,
|
1048
|
+
int protocol,
|
1049
|
+
unsigned file_index,
|
1050
|
+
unsigned int flags)
|
1051
|
+
{
|
1052
|
+
io_uring_prep_rw(IORING_OP_SOCKET, sqe, domain, NULL, protocol, type);
|
1053
|
+
sqe->rw_flags = flags;
|
1054
|
+
__io_uring_set_target_fixed_file(sqe, file_index);
|
1055
|
+
}
|
1056
|
+
|
1057
|
+
static inline void io_uring_prep_socket_direct_alloc(struct io_uring_sqe *sqe,
|
1058
|
+
int domain, int type, int protocol,
|
1059
|
+
unsigned int flags)
|
1060
|
+
{
|
1061
|
+
io_uring_prep_rw(IORING_OP_SOCKET, sqe, domain, NULL, protocol, type);
|
1062
|
+
sqe->rw_flags = flags;
|
1063
|
+
__io_uring_set_target_fixed_file(sqe, IORING_FILE_INDEX_ALLOC - 1);
|
1064
|
+
}
|
1065
|
+
|
719
1066
|
/*
|
720
1067
|
* Returns number of unconsumed (if SQPOLL) or unsubmitted entries exist in
|
721
1068
|
* the SQ ring
|
722
1069
|
*/
|
723
1070
|
static inline unsigned io_uring_sq_ready(const struct io_uring *ring)
|
724
1071
|
{
|
1072
|
+
unsigned khead = *ring->sq.khead;
|
1073
|
+
|
725
1074
|
/*
|
726
1075
|
* Without a barrier, we could miss an update and think the SQ wasn't
|
727
1076
|
* ready. We don't need the load acquire for non-SQPOLL since then we
|
728
1077
|
* drive updates.
|
729
1078
|
*/
|
730
1079
|
if (ring->flags & IORING_SETUP_SQPOLL)
|
731
|
-
|
1080
|
+
khead = io_uring_smp_load_acquire(ring->sq.khead);
|
732
1081
|
|
733
1082
|
/* always use real head, to avoid losing sync for short submit */
|
734
|
-
return ring->sq.sqe_tail -
|
1083
|
+
return ring->sq.sqe_tail - khead;
|
735
1084
|
}
|
736
1085
|
|
737
1086
|
/*
|
@@ -739,7 +1088,7 @@ static inline unsigned io_uring_sq_ready(const struct io_uring *ring)
|
|
739
1088
|
*/
|
740
1089
|
static inline unsigned io_uring_sq_space_left(const struct io_uring *ring)
|
741
1090
|
{
|
742
|
-
return
|
1091
|
+
return ring->sq.ring_entries - io_uring_sq_ready(ring);
|
743
1092
|
}
|
744
1093
|
|
745
1094
|
/*
|
@@ -767,6 +1116,15 @@ static inline unsigned io_uring_cq_ready(const struct io_uring *ring)
|
|
767
1116
|
return io_uring_smp_load_acquire(ring->cq.ktail) - *ring->cq.khead;
|
768
1117
|
}
|
769
1118
|
|
1119
|
+
/*
|
1120
|
+
* Returns true if there are overflow entries waiting to be flushed onto
|
1121
|
+
* the CQ ring
|
1122
|
+
*/
|
1123
|
+
static inline bool io_uring_cq_has_overflow(const struct io_uring *ring)
|
1124
|
+
{
|
1125
|
+
return IO_URING_READ_ONCE(*ring->sq.kflags) & IORING_SQ_CQ_OVERFLOW;
|
1126
|
+
}
|
1127
|
+
|
770
1128
|
/*
|
771
1129
|
* Returns true if the eventfd notification is currently enabled
|
772
1130
|
*/
|
@@ -817,6 +1175,11 @@ static inline int io_uring_wait_cqe_nr(struct io_uring *ring,
|
|
817
1175
|
return __io_uring_get_cqe(ring, cqe_ptr, 0, wait_nr, NULL);
|
818
1176
|
}
|
819
1177
|
|
1178
|
+
/*
|
1179
|
+
* Internal helper, don't use directly in applications. Use one of the
|
1180
|
+
* "official" versions of this, io_uring_peek_cqe(), io_uring_wait_cqe(),
|
1181
|
+
* or io_uring_wait_cqes*().
|
1182
|
+
*/
|
820
1183
|
static inline int __io_uring_peek_cqe(struct io_uring *ring,
|
821
1184
|
struct io_uring_cqe **cqe_ptr,
|
822
1185
|
unsigned *nr_available)
|
@@ -824,7 +1187,11 @@ static inline int __io_uring_peek_cqe(struct io_uring *ring,
|
|
824
1187
|
struct io_uring_cqe *cqe;
|
825
1188
|
int err = 0;
|
826
1189
|
unsigned available;
|
827
|
-
unsigned mask =
|
1190
|
+
unsigned mask = ring->cq.ring_mask;
|
1191
|
+
int shift = 0;
|
1192
|
+
|
1193
|
+
if (ring->flags & IORING_SETUP_CQE32)
|
1194
|
+
shift = 1;
|
828
1195
|
|
829
1196
|
do {
|
830
1197
|
unsigned tail = io_uring_smp_load_acquire(ring->cq.ktail);
|
@@ -835,7 +1202,7 @@ static inline int __io_uring_peek_cqe(struct io_uring *ring,
|
|
835
1202
|
if (!available)
|
836
1203
|
break;
|
837
1204
|
|
838
|
-
cqe = &ring->cq.cqes[head & mask];
|
1205
|
+
cqe = &ring->cq.cqes[(head & mask) << shift];
|
839
1206
|
if (!(ring->features & IORING_FEAT_EXT_ARG) &&
|
840
1207
|
cqe->user_data == LIBURING_UDATA_TIMEOUT) {
|
841
1208
|
if (cqe->res < 0)
|
@@ -862,7 +1229,7 @@ static inline int __io_uring_peek_cqe(struct io_uring *ring,
|
|
862
1229
|
static inline int io_uring_peek_cqe(struct io_uring *ring,
|
863
1230
|
struct io_uring_cqe **cqe_ptr)
|
864
1231
|
{
|
865
|
-
if (__io_uring_peek_cqe(ring, cqe_ptr, NULL))
|
1232
|
+
if (!__io_uring_peek_cqe(ring, cqe_ptr, NULL) && *cqe_ptr)
|
866
1233
|
return 0;
|
867
1234
|
|
868
1235
|
return io_uring_wait_cqe_nr(ring, cqe_ptr, 0);
|
@@ -875,7 +1242,7 @@ static inline int io_uring_peek_cqe(struct io_uring *ring,
|
|
875
1242
|
static inline int io_uring_wait_cqe(struct io_uring *ring,
|
876
1243
|
struct io_uring_cqe **cqe_ptr)
|
877
1244
|
{
|
878
|
-
if (__io_uring_peek_cqe(ring, cqe_ptr, NULL))
|
1245
|
+
if (!__io_uring_peek_cqe(ring, cqe_ptr, NULL) && *cqe_ptr)
|
879
1246
|
return 0;
|
880
1247
|
|
881
1248
|
return io_uring_wait_cqe_nr(ring, cqe_ptr, 1);
|
@@ -891,15 +1258,81 @@ static inline int io_uring_wait_cqe(struct io_uring *ring,
|
|
891
1258
|
static inline struct io_uring_sqe *_io_uring_get_sqe(struct io_uring *ring)
|
892
1259
|
{
|
893
1260
|
struct io_uring_sq *sq = &ring->sq;
|
894
|
-
unsigned int head =
|
895
|
-
|
896
|
-
struct io_uring_sqe *sqe = NULL;
|
1261
|
+
unsigned int head, next = sq->sqe_tail + 1;
|
1262
|
+
int shift = 0;
|
897
1263
|
|
898
|
-
if (
|
899
|
-
|
1264
|
+
if (ring->flags & IORING_SETUP_SQE128)
|
1265
|
+
shift = 1;
|
1266
|
+
if (!(ring->flags & IORING_SETUP_SQPOLL))
|
1267
|
+
head = IO_URING_READ_ONCE(*sq->khead);
|
1268
|
+
else
|
1269
|
+
head = io_uring_smp_load_acquire(sq->khead);
|
1270
|
+
|
1271
|
+
if (next - head <= sq->ring_entries) {
|
1272
|
+
struct io_uring_sqe *sqe;
|
1273
|
+
|
1274
|
+
sqe = &sq->sqes[(sq->sqe_tail & sq->ring_mask) << shift];
|
900
1275
|
sq->sqe_tail = next;
|
1276
|
+
return sqe;
|
901
1277
|
}
|
902
|
-
|
1278
|
+
|
1279
|
+
return NULL;
|
1280
|
+
}
|
1281
|
+
|
1282
|
+
/*
|
1283
|
+
* Return the appropriate mask for a buffer ring of size 'ring_entries'
|
1284
|
+
*/
|
1285
|
+
static inline int io_uring_buf_ring_mask(__u32 ring_entries)
|
1286
|
+
{
|
1287
|
+
return ring_entries - 1;
|
1288
|
+
}
|
1289
|
+
|
1290
|
+
static inline void io_uring_buf_ring_init(struct io_uring_buf_ring *br)
|
1291
|
+
{
|
1292
|
+
br->tail = 0;
|
1293
|
+
}
|
1294
|
+
|
1295
|
+
/*
|
1296
|
+
* Assign 'buf' with the addr/len/buffer ID supplied
|
1297
|
+
*/
|
1298
|
+
static inline void io_uring_buf_ring_add(struct io_uring_buf_ring *br,
|
1299
|
+
void *addr, unsigned int len,
|
1300
|
+
unsigned short bid, int mask,
|
1301
|
+
int buf_offset)
|
1302
|
+
{
|
1303
|
+
struct io_uring_buf *buf = &br->bufs[(br->tail + buf_offset) & mask];
|
1304
|
+
|
1305
|
+
buf->addr = (unsigned long) (uintptr_t) addr;
|
1306
|
+
buf->len = len;
|
1307
|
+
buf->bid = bid;
|
1308
|
+
}
|
1309
|
+
|
1310
|
+
/*
|
1311
|
+
* Make 'count' new buffers visible to the kernel. Called after
|
1312
|
+
* io_uring_buf_ring_add() has been called 'count' times to fill in new
|
1313
|
+
* buffers.
|
1314
|
+
*/
|
1315
|
+
static inline void io_uring_buf_ring_advance(struct io_uring_buf_ring *br,
|
1316
|
+
int count)
|
1317
|
+
{
|
1318
|
+
unsigned short new_tail = br->tail + count;
|
1319
|
+
|
1320
|
+
io_uring_smp_store_release(&br->tail, new_tail);
|
1321
|
+
}
|
1322
|
+
|
1323
|
+
/*
|
1324
|
+
* Make 'count' new buffers visible to the kernel while at the same time
|
1325
|
+
* advancing the CQ ring seen entries. This can be used when the application
|
1326
|
+
* is using ring provided buffers and returns buffers while processing CQEs,
|
1327
|
+
* avoiding an extra atomic when needing to increment both the CQ ring and
|
1328
|
+
* the ring buffer index at the same time.
|
1329
|
+
*/
|
1330
|
+
static inline void io_uring_buf_ring_cq_advance(struct io_uring *ring,
|
1331
|
+
struct io_uring_buf_ring *br,
|
1332
|
+
int count)
|
1333
|
+
{
|
1334
|
+
br->tail += count;
|
1335
|
+
io_uring_cq_advance(ring, count);
|
903
1336
|
}
|
904
1337
|
|
905
1338
|
#ifndef LIBURING_INTERNAL
|