polyphony 0.45.0 → 0.46.0

data/examples/core/{xx-recurrent-timer.rb → recurrent-timer.rb}

@@ -3,7 +3,7 @@
 require 'bundler/setup'
 require 'polyphony'
 
-move_on_after(3) do
+move_on_after(3.1) do
   puts 'Start...'
   every(1) do
     puts Time.now

data/examples/core/{xx-resource_delegate.rb → resource_delegate.rb}

@@ -10,7 +10,7 @@ class Number
 
   def greet(other)
     puts "You are number #{other}, I am number #{@id}"
-    sleep(0.05 + rand * 0.2)
+    sleep rand(0.2..0.3)
   end
 end
 
@@ -25,7 +25,6 @@ def meet(number)
   end
 end
 
-3.times { |x| spin { meet(x) } }
+(4..10).each { |x| spin { meet(x) } }
 
-t0 = Time.now
-every(10) { puts "uptime: #{Time.now - t0}" }
+sleep 1

data/examples/core/{01-spinning-up-fibers.rb → spin.rb}

@@ -14,5 +14,5 @@ end
 spin { nap(:a, 1) }
 spin { nap(:b, 2) }
 
-# Calling suspend will block until all child fibers have terminated
+# Calling suspend will block until no work is left to do
 suspend

data/examples/core/{xx-spin_error_backtrace.rb → spin_error_backtrace.rb}

@@ -9,7 +9,7 @@ end
 
 def deferred_error(t)
   puts "deferred_error"
-  spin { de2(t) }
+  spin { de2(t) }.await
 end
 
 def de2(t)

data/examples/core/{xx-supervise-process.rb → supervise-process.rb}

@@ -8,7 +8,7 @@ Exception.__disable_sanitized_backtrace__ = true
 supervisor = spin do
   puts "parent pid #{Process.pid}"
 
-  Polyphony::ProcessSupervisor.supervise do
+  Polyphony.watch_process do
     puts "child pid #{Process.pid}"
     puts "go to sleep"
     sleep 5
@@ -22,9 +22,12 @@ supervisor = spin do
 end
 
 begin
+  spin do
+    sleep 2.5
+    Process.kill('TERM', Process.pid)
+  end
+  supervisor.await
+rescue SystemExit
+  supervisor.terminate
   supervisor.await
-rescue Interrupt
-  exit!
-  # supervisor.terminate
-  # supervisor.await
 end

data/examples/core/supervisor.rb

@@ -0,0 +1,20 @@
+# frozen_string_literal: true
+
+require 'bundler/setup'
+require 'polyphony'
+
+def my_sleep(t)
+  puts "#{t} start"
+  sleep(t)
+  puts "#{t} done"
+end
+
+spin { my_sleep(1) }
+spin { my_sleep(2) }
+spin { my_sleep(3) }
+spin { puts "fiber count: #{Fiber.current.children.count}" }
+snooze
+
+puts "#{Time.now} supervising..."
+supervise
+puts "#{Time.now} done supervising"

data/examples/core/{xx-worker-thread.rb → worker-thread.rb}

@@ -5,9 +5,9 @@ require 'polyphony'
 
 def do_work(client)
   result = yield
-  client.schedule(result)
+  client << result
 rescue Exception => e
-  client.schedule(e)
+  client << e
 end
 
 $worker = Thread.new do

data/examples/io/{xx-echo_client.rb → echo_client.rb}

@@ -15,7 +15,7 @@ end
 
 reader = spin do
   puts 'received from echo server:'
-  while (data = socket.readpartial(8192))
+  socket.read_loop do |data|
     STDOUT << data
   end
 end

data/examples/io/{xx-echo_client_from_stdin.rb → echo_client_from_stdin.rb}

@@ -11,8 +11,8 @@ writer = spin do
   end
 end
 
-spin do
-  while (data = socket.readpartial(8192))
+reader = spin do
+  socket.read_loop do |data|
     STDOUT << 'received: ' + data
   end
   writer.interrupt

data/examples/io/{xx-echo_pipe.rb → echo_pipe.rb}

@@ -11,6 +11,6 @@ spin do
   o.close
 end
 
-while (data = i.readpartial(8192))
+i.read_loop do |data|
   STDOUT << "You said: #{data}"
 end

data/examples/io/{xx-echo_server_with_timeout.rb → echo_server_with_timeout.rb}

@@ -17,7 +17,7 @@ begin
         scope.when_cancelled do
           client.write "Disconnecting due to inactivity\n"
         end
-        while (data = client.readpartial(8192))
+        client.read_loop do |data|
           scope.reset_timeout
           client.write "You said: #{data}"
         end

data/examples/io/{xx-httparty.rb → httparty.rb}

@@ -16,23 +16,14 @@ end
 zones = %w{
   Europe/London Europe/Paris Europe/Bucharest America/New_York Asia/Bangkok
 }
-# zones.each do |tzone|
-#   spin do
-#     time = get_time(tzone)
-#     puts "Time in #{tzone}: #{time}"
-#   end
-# end
-
-# suspend
 
 def get_times(zones)
-  Polyphony::Supervisor.new do |s|
-    zones.each do |tzone|
-      s.spin { [tzone, get_time(tzone)] }
-    end
+  fibers = zones.map do |tzone|
+    spin { [tzone, get_time(tzone)] }
   end
+  Fiber.await(*fibers)
 end
 
-get_times(zones).await.each do |tzone, time|
+get_times(zones).each do |tzone, time|
   puts "Time in #{tzone}: #{time}"
 end

data/examples/io/raw.rb

@@ -0,0 +1,14 @@
+# frozen_string_literal: true
+
+require 'bundler/setup'
+require 'polyphony'
+require 'io/console'
+
+c = STDIN.raw(min: 1, tim: 0, &:getbyte)
+p result: c
+exit
+
+puts '?' * 40
+c = STDIN.getbyte
+puts '*' * 40
+p c

data/examples/io/reline.rb

@@ -0,0 +1,18 @@
+# frozen_string_literal: true
+
+require 'bundler/setup'
+require 'polyphony'
+require 'polyphony/adapters/readline'
+require 'pry'
+
+$counter = 0
+timer = spin do
+  throttled_loop(5) do
+    $counter += 1
+  end
+end
+
+at_exit { timer.stop }
+
+puts 'try typing $counter to see the counter incremented in the background'
+binding.pry

data/examples/io/{xx-system.rb → system.rb}

@@ -7,5 +7,5 @@ timer = spin do
   throttled_loop(5) { STDOUT << '.' }
 end
 
-puts system('ruby -e "sleep 1; puts :done; STDOUT.close"')
+puts system('ruby -e "puts :sleeping; STDOUT.flush; sleep 1; puts :done"')
 timer.stop

data/examples/io/tunnel.rb

@@ -3,6 +3,11 @@
 require 'bundler/setup'
 require 'polyphony'
 
+if ARGV.size < 2
+  puts "Usage: ruby examples/tunnel.rb <port1> <port2>"
+  exit
+end
+
 Ports = ARGV[0..1]
 EndPoints = []
 
@@ -24,7 +29,7 @@ def endpoint_loop(idx, peer_idx)
     conn.binmode
     EndPoints[idx] = conn
     log "Client connected on port #{port} (#{conn.remote_address.inspect})"
-    while data = conn.readpartial(8192)
+    conn.read_loop do |data|
       peer = EndPoints[peer_idx]
       if peer
         peer << data

data/examples/performance/fiber_transfer.rb

@@ -6,7 +6,8 @@ class Fiber
   attr_accessor :next
 end
 
-# This program shows how the performance 
+# This program shows how the performance of Fiber.transfer degrades as the fiber
+# count increases
 
 def run(num_fibers)
   count = 0

data/examples/performance/fs_read.rb

@@ -2,7 +2,7 @@
 
 require 'bundler/setup'
 require 'polyphony'
-require 'polyphony/fs'
+require 'polyphony/adapters/fs'
 
 def raw_read_file(x)
   t0 = Time.now
@@ -14,7 +14,7 @@ def threaded_read_file(x, y)
   t0 = Time.now
   threads = []
   y.times do
-    threads << Thread.new { x.times { IO.orig_read(PATH) } }
+    threads << Thread.new { x.times { IO.orig_read(__FILE__) } }
   end
   threads.each(&:join)
   puts "threaded_read_file: #{Time.now - t0}"
@@ -22,11 +22,10 @@ end
 
 def thread_pool_read_file(x, y)
   t0 = Time.now
-  supervise do |s|
-    y.times do
-      s.spin { x.times { IO.read(PATH) } }
-    end
+  y.times do
+    spin { x.times { IO.read(__FILE__) } }
   end
+  Fiber.current.await_all_children
   puts "thread_pool_read_file: #{Time.now - t0}"
 end
 

data/examples/performance/multi_snooze.rb

@@ -18,7 +18,6 @@ def bm(fibers, iterations)
   Fiber.current.await_all_children
   dt = Time.now - t0
   puts "#{[fibers, iterations].inspect} setup: #{t0 - t_pre}s count: #{count} #{count / dt.to_f}/s"
-  Thread.current.run_queue_trace
 end
 
 GC.disable

data/examples/{io/xx-switch.rb → performance/switch.rb}

@@ -12,4 +12,5 @@ end
 t0 = Time.now
 X.times { f.transfer }
 dt = Time.now - t0
-puts "#{X / dt.to_f}/s"
+puts "#{X / dt.to_f}/s"
+puts fs.size

data/examples/performance/thread-vs-fiber/{xx-httparty_multi.rb → httparty_multi.rb}

@@ -21,11 +21,10 @@ end
 t0 = Time.now
 results = []
 move_on_after(3) do
-  supervise do |s|
-    10.times do
-      s.spin { get_time(results) }
-    end
+  10.times do
+    spin { get_time(results) }
   end
+  supervise
   puts 'done'
 end
 

data/examples/performance/thread-vs-fiber/polyphony_mt_server.rb

@@ -24,7 +24,7 @@ def handle_client(socket)
   parser.on_message_complete = proc do |env|
     reqs << Object.new # parser
   end
-  while (data = socket.readpartial(8192)) do
+  socket.read_loop do |data|
     parser << data
     while (req = reqs.shift)
       handle_request(socket, req)

data/examples/performance/thread-vs-fiber/polyphony_server.rb

@@ -13,12 +13,11 @@ def handle_client(socket)
   parser.on_message_complete = proc do |env|
     reqs << Object.new # parser
   end
-  while (data = socket.readpartial(8192)) do
+  socket.recv_loop do |data|
     parser << data
     while (req = reqs.shift)
       handle_request(socket, req)
       req = nil
-      snooze
     end
   end
 rescue IOError, SystemCallError => e

data/examples/performance/thread-vs-fiber/threaded_server.rb

@@ -11,11 +11,7 @@ def handle_client(client)
       headers = "Content-Length: #{data.bytesize}\r\n"
       client.write "HTTP/1.1 #{status_code}\r\n#{headers}\r\n#{data}"
     end
-    loop do
-      while data = client.readpartial(8192) rescue nil
-        parser << data
-      end
-    end
+    client.read_loop { |data| parser << data }
     client.close
   end
 end

data/examples/performance/thread_pool_perf.rb

@@ -10,7 +10,7 @@ def lengthy_op
   # Digest::SHA256.digest(IO.read('doc/Promise.html'))
 end
 
-X = 100
+X = 10000
 
 def compare_performance
   t0 = Time.now
@@ -35,20 +35,19 @@ def compare_performance
 
     acc = 0
     count = 0
-    10.times do |_i|
+    1.times do |_i|
       t0 = Time.now
-      supervise do |s|
-        X.times do
-          s.spin { Polyphony::ThreadPool.process { lengthy_op } }
-        end
+      X.times do
+        spin { Polyphony::ThreadPool.process { lengthy_op } }
       end
+      Fiber.current.await_all_children
       thread_pool_perf = X / (Time.now - t0)
       acc += thread_pool_perf
       count += 1
     end
     avg_perf = acc / count
     puts format(
-      'avg thread pool performance: %g (X %0.2f)',
+      'spin X thread pool performance: %g (X %0.2f)',
       avg_perf,
       avg_perf / native_perf
     )

data/ext/liburing/liburing.h

@@ -0,0 +1,585 @@
+/* SPDX-License-Identifier: MIT */
+#ifndef LIB_URING_H
+#define LIB_URING_H
+
+#include <sys/socket.h>
+#include <sys/uio.h>
+#include <sys/stat.h>
+#include <errno.h>
+#include <signal.h>
+#include <stdbool.h>
+#include <inttypes.h>
+#include <time.h>
+#include <linux/swab.h>
+#include "liburing/compat.h"
+#include "liburing/io_uring.h"
+#include "liburing/barrier.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+/*
+ * Library interface to io_uring
+ */
+struct io_uring_sq {
+	unsigned *khead;
+	unsigned *ktail;
+	unsigned *kring_mask;
+	unsigned *kring_entries;
+	unsigned *kflags;
+	unsigned *kdropped;
+	unsigned *array;
+	struct io_uring_sqe *sqes;
+
+	unsigned sqe_head;
+	unsigned sqe_tail;
+
+	size_t ring_sz;
+	void *ring_ptr;
+
+	unsigned pad[4];
+};
+
+struct io_uring_cq {
+	unsigned *khead;
+	unsigned *ktail;
+	unsigned *kring_mask;
+	unsigned *kring_entries;
+	unsigned *kflags;
+	unsigned *koverflow;
+	struct io_uring_cqe *cqes;
+
+	size_t ring_sz;
+	void *ring_ptr;
+
+	unsigned pad[4];
+};
+
+struct io_uring {
+	struct io_uring_sq sq;
+	struct io_uring_cq cq;
+	unsigned flags;
+	int ring_fd;
+
+	unsigned pad[4];
+};
+
+/*
+ * Library interface
+ */
+
+/*
+ * return an allocated io_uring_probe structure, or NULL if probe fails (for
+ * example, if it is not available). The caller is responsible for freeing it
+ */
+extern struct io_uring_probe *io_uring_get_probe_ring(struct io_uring *ring);
+/* same as io_uring_get_probe_ring, but takes care of ring init and teardown */
+extern struct io_uring_probe *io_uring_get_probe(void);
+
+/*
+ * frees a probe allocated through io_uring_get_probe() or
+ * io_uring_get_probe_ring()
+ */
+extern void io_uring_free_probe(struct io_uring_probe *probe);
+
+static inline int io_uring_opcode_supported(struct io_uring_probe *p, int op)
+{
+	if (op > p->last_op)
+		return 0;
+	return (p->ops[op].flags & IO_URING_OP_SUPPORTED) != 0;
+}
+
+extern int io_uring_queue_init_params(unsigned entries, struct io_uring *ring,
+	struct io_uring_params *p);
+extern int io_uring_queue_init(unsigned entries, struct io_uring *ring,
+	unsigned flags);
+extern int io_uring_queue_mmap(int fd, struct io_uring_params *p,
+	struct io_uring *ring);
+extern int io_uring_ring_dontfork(struct io_uring *ring);
+extern void io_uring_queue_exit(struct io_uring *ring);
+unsigned io_uring_peek_batch_cqe(struct io_uring *ring,
+	struct io_uring_cqe **cqes, unsigned count);
+extern int io_uring_wait_cqes(struct io_uring *ring,
+	struct io_uring_cqe **cqe_ptr, unsigned wait_nr,
+	struct __kernel_timespec *ts, sigset_t *sigmask);
+extern int io_uring_wait_cqe_timeout(struct io_uring *ring,
+	struct io_uring_cqe **cqe_ptr, struct __kernel_timespec *ts);
+extern int io_uring_submit(struct io_uring *ring);
+extern int io_uring_submit_and_wait(struct io_uring *ring, unsigned wait_nr);
+extern struct io_uring_sqe *io_uring_get_sqe(struct io_uring *ring);
+
+extern int io_uring_register_buffers(struct io_uring *ring,
+					const struct iovec *iovecs,
+					unsigned nr_iovecs);
+extern int io_uring_unregister_buffers(struct io_uring *ring);
+extern int io_uring_register_files(struct io_uring *ring, const int *files,
+					unsigned nr_files);
+extern int io_uring_unregister_files(struct io_uring *ring);
+extern int io_uring_register_files_update(struct io_uring *ring, unsigned off,
+					int *files, unsigned nr_files);
+extern int io_uring_register_eventfd(struct io_uring *ring, int fd);
+extern int io_uring_register_eventfd_async(struct io_uring *ring, int fd);
+extern int io_uring_unregister_eventfd(struct io_uring *ring);
+extern int io_uring_register_probe(struct io_uring *ring,
+					struct io_uring_probe *p, unsigned nr);
+extern int io_uring_register_personality(struct io_uring *ring);
+extern int io_uring_unregister_personality(struct io_uring *ring, int id);
+extern int io_uring_register_restrictions(struct io_uring *ring,
+					  struct io_uring_restriction *res,
+					  unsigned int nr_res);
+extern int io_uring_enable_rings(struct io_uring *ring);
+extern int __io_uring_sqring_wait(struct io_uring *ring);
+
+/*
+ * Helper for the peek/wait single cqe functions. Exported because of that,
+ * but probably shouldn't be used directly in an application.
+ */
+extern int __io_uring_get_cqe(struct io_uring *ring,
+			      struct io_uring_cqe **cqe_ptr, unsigned submit,
+			      unsigned wait_nr, sigset_t *sigmask);
+
+#define LIBURING_UDATA_TIMEOUT	((__u64) -1)
+
+#define io_uring_for_each_cqe(ring, head, cqe)				\
+	/*								\
+	 * io_uring_smp_load_acquire() enforces the order of tail	\
+	 * and CQE reads.						\
+	 */								\
+	for (head = *(ring)->cq.khead;					\
+	     (cqe = (head != io_uring_smp_load_acquire((ring)->cq.ktail) ? \
+		&(ring)->cq.cqes[head & (*(ring)->cq.kring_mask)] : NULL)); \
+	     head++)							\
+
+/*
+ * Must be called after io_uring_for_each_cqe()
+ */
+static inline void io_uring_cq_advance(struct io_uring *ring,
+				       unsigned nr)
+{
+	if (nr) {
+		struct io_uring_cq *cq = &ring->cq;
+
+		/*
+		 * Ensure that the kernel only sees the new value of the head
+		 * index after the CQEs have been read.
+		 */
+		io_uring_smp_store_release(cq->khead, *cq->khead + nr);
+	}
+}
+
+/*
+ * Must be called after io_uring_{peek,wait}_cqe() after the cqe has
+ * been processed by the application.
+ */
+static inline void io_uring_cqe_seen(struct io_uring *ring,
+				     struct io_uring_cqe *cqe)
+{
+	if (cqe)
+		io_uring_cq_advance(ring, 1);
+}
+
+/*
+ * Command prep helpers
+ */
+static inline void io_uring_sqe_set_data(struct io_uring_sqe *sqe, void *data)
+{
+	sqe->user_data = (unsigned long) data;
+}
+
+static inline void *io_uring_cqe_get_data(const struct io_uring_cqe *cqe)
+{
+	return (void *) (uintptr_t) cqe->user_data;
+}
+
+static inline void io_uring_sqe_set_flags(struct io_uring_sqe *sqe,
+					  unsigned flags)
+{
+	sqe->flags = flags;
+}
+
+static inline void io_uring_prep_rw(int op, struct io_uring_sqe *sqe, int fd,
+				    const void *addr, unsigned len,
+				    __u64 offset)
+{
+	sqe->opcode = op;
+	sqe->flags = 0;
+	sqe->ioprio = 0;
+	sqe->fd = fd;
+	sqe->off = offset;
+	sqe->addr = (unsigned long) addr;
+	sqe->len = len;
+	sqe->rw_flags = 0;
+	sqe->user_data = 0;
+	sqe->__pad2[0] = sqe->__pad2[1] = sqe->__pad2[2] = 0;
+}
+
+static inline void io_uring_prep_splice(struct io_uring_sqe *sqe,
+					int fd_in, int64_t off_in,
+					int fd_out, int64_t off_out,
+					unsigned int nbytes,
+					unsigned int splice_flags)
+{
+	io_uring_prep_rw(IORING_OP_SPLICE, sqe, fd_out, NULL, nbytes, off_out);
+	sqe->splice_off_in = off_in;
+	sqe->splice_fd_in = fd_in;
+	sqe->splice_flags = splice_flags;
+}
+
+static inline void io_uring_prep_readv(struct io_uring_sqe *sqe, int fd,
+				       const struct iovec *iovecs,
+				       unsigned nr_vecs, off_t offset)
+{
+	io_uring_prep_rw(IORING_OP_READV, sqe, fd, iovecs, nr_vecs, offset);
+}
+
+static inline void io_uring_prep_read_fixed(struct io_uring_sqe *sqe, int fd,
+					    void *buf, unsigned nbytes,
+					    off_t offset, int buf_index)
+{
+	io_uring_prep_rw(IORING_OP_READ_FIXED, sqe, fd, buf, nbytes, offset);
+	sqe->buf_index = buf_index;
+}
+
+static inline void io_uring_prep_writev(struct io_uring_sqe *sqe, int fd,
+					const struct iovec *iovecs,
+					unsigned nr_vecs, off_t offset)
+{
+	io_uring_prep_rw(IORING_OP_WRITEV, sqe, fd, iovecs, nr_vecs, offset);
+}
+
+static inline void io_uring_prep_write_fixed(struct io_uring_sqe *sqe, int fd,
+					     const void *buf, unsigned nbytes,
+					     off_t offset, int buf_index)
+{
+	io_uring_prep_rw(IORING_OP_WRITE_FIXED, sqe, fd, buf, nbytes, offset);
+	sqe->buf_index = buf_index;
+}
+
+static inline void io_uring_prep_recvmsg(struct io_uring_sqe *sqe, int fd,
+					 struct msghdr *msg, unsigned flags)
+{
+	io_uring_prep_rw(IORING_OP_RECVMSG, sqe, fd, msg, 1, 0);
+	sqe->msg_flags = flags;
+}
+
+static inline void io_uring_prep_sendmsg(struct io_uring_sqe *sqe, int fd,
+					 const struct msghdr *msg, unsigned flags)
+{
+	io_uring_prep_rw(IORING_OP_SENDMSG, sqe, fd, msg, 1, 0);
+	sqe->msg_flags = flags;
+}
+
+static inline void io_uring_prep_poll_add(struct io_uring_sqe *sqe, int fd,
+					  unsigned poll_mask)
+{
+	io_uring_prep_rw(IORING_OP_POLL_ADD, sqe, fd, NULL, 0, 0);
+#if __BYTE_ORDER == __BIG_ENDIAN
+	poll_mask = __swahw32(poll_mask);
+#endif
+	sqe->poll32_events = poll_mask;
+}
+
+static inline void io_uring_prep_poll_remove(struct io_uring_sqe *sqe,
+					     void *user_data)
+{
+	io_uring_prep_rw(IORING_OP_POLL_REMOVE, sqe, -1, user_data, 0, 0);
+}
+
+static inline void io_uring_prep_fsync(struct io_uring_sqe *sqe, int fd,
+				       unsigned fsync_flags)
+{
+	io_uring_prep_rw(IORING_OP_FSYNC, sqe, fd, NULL, 0, 0);
+	sqe->fsync_flags = fsync_flags;
+}
+
+static inline void io_uring_prep_nop(struct io_uring_sqe *sqe)
+{
+	io_uring_prep_rw(IORING_OP_NOP, sqe, -1, NULL, 0, 0);
+}
+
+static inline void io_uring_prep_timeout(struct io_uring_sqe *sqe,
+					 struct __kernel_timespec *ts,
+					 unsigned count, unsigned flags)
+{
+	io_uring_prep_rw(IORING_OP_TIMEOUT, sqe, -1, ts, 1, count);
+	sqe->timeout_flags = flags;
+}
+
+static inline void io_uring_prep_timeout_remove(struct io_uring_sqe *sqe,
+						__u64 user_data, unsigned flags)
+{
+	io_uring_prep_rw(IORING_OP_TIMEOUT_REMOVE, sqe, -1,
+				(void *)(unsigned long)user_data, 0, 0);
+	sqe->timeout_flags = flags;
+}
+
+static inline void io_uring_prep_accept(struct io_uring_sqe *sqe, int fd,
+					struct sockaddr *addr,
+					socklen_t *addrlen, int flags)
+{
+	io_uring_prep_rw(IORING_OP_ACCEPT, sqe, fd, addr, 0,
+				(__u64) (unsigned long) addrlen);
+	sqe->accept_flags = flags;
+}
+
+static inline void io_uring_prep_cancel(struct io_uring_sqe *sqe, void *user_data,
+					int flags)
+{
+	io_uring_prep_rw(IORING_OP_ASYNC_CANCEL, sqe, -1, user_data, 0, 0);
+	sqe->cancel_flags = flags;
+}
+
+static inline void io_uring_prep_link_timeout(struct io_uring_sqe *sqe,
+					      struct __kernel_timespec *ts,
+					      unsigned flags)
+{
+	io_uring_prep_rw(IORING_OP_LINK_TIMEOUT, sqe, -1, ts, 1, 0);
+	sqe->timeout_flags = flags;
+}
+
+static inline void io_uring_prep_connect(struct io_uring_sqe *sqe, int fd,
+					 const struct sockaddr *addr,
+					 socklen_t addrlen)
+{
+	io_uring_prep_rw(IORING_OP_CONNECT, sqe, fd, addr, 0, addrlen);
+}
+
+static inline void io_uring_prep_files_update(struct io_uring_sqe *sqe,
+					      int *fds, unsigned nr_fds,
+					      int offset)
+{
+	io_uring_prep_rw(IORING_OP_FILES_UPDATE, sqe, -1, fds, nr_fds, offset);
+}
+
+static inline void io_uring_prep_fallocate(struct io_uring_sqe *sqe, int fd,
+					   int mode, off_t offset, off_t len)
+{
+
+	io_uring_prep_rw(IORING_OP_FALLOCATE, sqe, fd,
+			(const uintptr_t *) (unsigned long) len, mode, offset);
+}
+
+static inline void io_uring_prep_openat(struct io_uring_sqe *sqe, int dfd,
+					const char *path, int flags, mode_t mode)
+{
+	io_uring_prep_rw(IORING_OP_OPENAT, sqe, dfd, path, mode, 0);
+	sqe->open_flags = flags;
+}
+
+static inline void io_uring_prep_close(struct io_uring_sqe *sqe, int fd)
+{
+	io_uring_prep_rw(IORING_OP_CLOSE, sqe, fd, NULL, 0, 0);
+}
+
+static inline void io_uring_prep_read(struct io_uring_sqe *sqe, int fd,
+				      void *buf, unsigned nbytes, off_t offset)
+{
+	io_uring_prep_rw(IORING_OP_READ, sqe, fd, buf, nbytes, offset);
+}
+
+static inline void io_uring_prep_write(struct io_uring_sqe *sqe, int fd,
+				       const void *buf, unsigned nbytes, off_t offset)
+{
+	io_uring_prep_rw(IORING_OP_WRITE, sqe, fd, buf, nbytes, offset);
+}
+
+struct statx;
+static inline void io_uring_prep_statx(struct io_uring_sqe *sqe, int dfd,
+				const char *path, int flags, unsigned mask,
+				struct statx *statxbuf)
+{
+	io_uring_prep_rw(IORING_OP_STATX, sqe, dfd, path, mask,
+				(__u64) (unsigned long) statxbuf);
+	sqe->statx_flags = flags;
+}
+
+static inline void io_uring_prep_fadvise(struct io_uring_sqe *sqe, int fd,
+					 off_t offset, off_t len, int advice)
+{
+	io_uring_prep_rw(IORING_OP_FADVISE, sqe, fd, NULL, len, offset);
+	sqe->fadvise_advice = advice;
+}
+
+static inline void io_uring_prep_madvise(struct io_uring_sqe *sqe, void *addr,
+					 off_t length, int advice)
+{
+	io_uring_prep_rw(IORING_OP_MADVISE, sqe, -1, addr, length, 0);
+	sqe->fadvise_advice = advice;
+}
+
+static inline void io_uring_prep_send(struct io_uring_sqe *sqe, int sockfd,
+				      const void *buf, size_t len, int flags)
+{
+	io_uring_prep_rw(IORING_OP_SEND, sqe, sockfd, buf, len, 0);
+	sqe->msg_flags = flags;
+}
+
+static inline void io_uring_prep_recv(struct io_uring_sqe *sqe, int sockfd,
+				      void *buf, size_t len, int flags)
+{
+	io_uring_prep_rw(IORING_OP_RECV, sqe, sockfd, buf, len, 0);
+	sqe->msg_flags = flags;
+}
+
+static inline void io_uring_prep_openat2(struct io_uring_sqe *sqe, int dfd,
+					const char *path, struct open_how *how)
+{
+	io_uring_prep_rw(IORING_OP_OPENAT2, sqe, dfd, path, sizeof(*how),
+				(uint64_t) (uintptr_t) how);
+}
+
+struct epoll_event;
+static inline void io_uring_prep_epoll_ctl(struct io_uring_sqe *sqe, int epfd,
+					   int fd, int op,
+					   struct epoll_event *ev)
+{
+	io_uring_prep_rw(IORING_OP_EPOLL_CTL, sqe, epfd, ev, op, fd);
+}
+
+static inline void io_uring_prep_provide_buffers(struct io_uring_sqe *sqe,
+						 void *addr, int len, int nr,
+						 int bgid, int bid)
+{
+	io_uring_prep_rw(IORING_OP_PROVIDE_BUFFERS, sqe, nr, addr, len, bid);
+	sqe->buf_group = bgid;
+}
+
+static inline void io_uring_prep_remove_buffers(struct io_uring_sqe *sqe,
+						int nr, int bgid)
+{
+	io_uring_prep_rw(IORING_OP_REMOVE_BUFFERS, sqe, nr, NULL, 0, 0);
+	sqe->buf_group = bgid;
+}
+
+static inline void io_uring_prep_shutdown(struct io_uring_sqe *sqe, int fd,
+					  int how)
+{
+	io_uring_prep_rw(IORING_OP_SHUTDOWN, sqe, fd, NULL, how, 0);
+}
+
+/*
+ * Returns number of unconsumed (if SQPOLL) or unsubmitted entries exist in
+ * the SQ ring
+ */
+static inline unsigned io_uring_sq_ready(struct io_uring *ring)
+{
+	/*
+	 * Without a barrier, we could miss an update and think the SQ wasn't ready.
+	 * We don't need the load acquire for non-SQPOLL since then we drive updates.
+	 */
+	if (ring->flags & IORING_SETUP_SQPOLL)
+		return ring->sq.sqe_tail - io_uring_smp_load_acquire(ring->sq.khead);
+
+	/* always use real head, to avoid losing sync for short submit */
+	return ring->sq.sqe_tail - *ring->sq.khead;
+}
+
+/*
+ * Returns how much space is left in the SQ ring.
+ */
+static inline unsigned io_uring_sq_space_left(struct io_uring *ring)
+{
+	return *ring->sq.kring_entries - io_uring_sq_ready(ring);
+}
+
+/*
+ * Only applicable when using SQPOLL - allows the caller to wait for space
+ * to free up in the SQ ring, which happens when the kernel side thread has
+ * consumed one or more entries. If the SQ ring is currently non-full, no
+ * action is taken. Note: may return -EINVAL if the kernel doesn't support
+ * this feature.
+ */
+static inline int io_uring_sqring_wait(struct io_uring *ring)
+{
+	if (!(ring->flags & IORING_SETUP_SQPOLL))
+		return 0;
+	if (io_uring_sq_space_left(ring))
+		return 0;
+
+	return __io_uring_sqring_wait(ring);
+}
+
+/*
+ * Returns how many unconsumed entries are ready in the CQ ring
+ */
+static inline unsigned io_uring_cq_ready(struct io_uring *ring)
+{
+	return io_uring_smp_load_acquire(ring->cq.ktail) - *ring->cq.khead;
+}
+
+/*
+ * Returns true if the eventfd notification is currently enabled
+ */
+static inline bool io_uring_cq_eventfd_enabled(struct io_uring *ring)
+{
+	if (!ring->cq.kflags)
+		return true;
+
+	return !(*ring->cq.kflags & IORING_CQ_EVENTFD_DISABLED);
+}
+
+/*
+ * Toggle eventfd notification on or off, if an eventfd is registered with
+ * the ring.
+ */
+static inline int io_uring_cq_eventfd_toggle(struct io_uring *ring,
+					     bool enabled)
+{
+	uint32_t flags;
+
+	if (!!enabled == io_uring_cq_eventfd_enabled(ring))
+		return 0;
+
+	if (!ring->cq.kflags)
+		return -EOPNOTSUPP;
+
+	flags = *ring->cq.kflags;
+
+	if (enabled)
+		flags &= ~IORING_CQ_EVENTFD_DISABLED;
+	else
+		flags |= IORING_CQ_EVENTFD_DISABLED;
+
+	IO_URING_WRITE_ONCE(*ring->cq.kflags, flags);
+
+	return 0;
+}
+
+/*
+ * Return an IO completion, waiting for 'wait_nr' completions if one isn't
+ * readily available. Returns 0 with cqe_ptr filled in on success, -errno on
+ * failure.
+ */
+static inline int io_uring_wait_cqe_nr(struct io_uring *ring,
+				      struct io_uring_cqe **cqe_ptr,
+				      unsigned wait_nr)
+{
+	return __io_uring_get_cqe(ring, cqe_ptr, 0, wait_nr, NULL);
+}
+
+/*
+ * Return an IO completion, if one is readily available. Returns 0 with
+ * cqe_ptr filled in on success, -errno on failure.
+ */
+static inline int io_uring_peek_cqe(struct io_uring *ring,
+				    struct io_uring_cqe **cqe_ptr)
+{
+	return io_uring_wait_cqe_nr(ring, cqe_ptr, 0);
+}
+
+/*
+ * Return an IO completion, waiting for it if necessary. Returns 0 with
+ * cqe_ptr filled in on success, -errno on failure.
+ */
+static inline int io_uring_wait_cqe(struct io_uring *ring,
+				    struct io_uring_cqe **cqe_ptr)
+{
+	return io_uring_wait_cqe_nr(ring, cqe_ptr, 1);
+}
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif