polyphony 0.93 → 0.95

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (312) hide show
  1. checksums.yaml +4 -4
  2. data/.github/workflows/test.yml +2 -2
  3. data/.gitignore +3 -3
  4. data/CHANGELOG.md +14 -0
  5. data/docs/api-reference/fiber.md +2 -2
  6. data/docs/api-reference/object.md +3 -3
  7. data/docs/main-concepts/exception-handling.md +2 -2
  8. data/examples/pipes/echo_server.rb +1 -1
  9. data/examples/pipes/http_server.rb +33 -0
  10. data/ext/polyphony/backend_common.c +53 -8
  11. data/ext/polyphony/backend_common.h +21 -13
  12. data/ext/polyphony/backend_io_uring.c +85 -147
  13. data/ext/polyphony/backend_libev.c +58 -89
  14. data/ext/polyphony/event.c +1 -1
  15. data/ext/polyphony/extconf.rb +7 -5
  16. data/ext/polyphony/fiber.c +5 -13
  17. data/ext/polyphony/io_extensions.c +68 -68
  18. data/ext/polyphony/pipe.c +1 -1
  19. data/ext/polyphony/polyphony.c +23 -23
  20. data/ext/polyphony/polyphony.h +0 -9
  21. data/ext/polyphony/polyphony_ext.c +1 -1
  22. data/ext/polyphony/queue.c +1 -1
  23. data/ext/polyphony/ring_buffer.c +1 -0
  24. data/ext/polyphony/socket_extensions.c +1 -1
  25. data/ext/polyphony/thread.c +1 -1
  26. data/lib/polyphony/extensions/enumerator.rb +16 -0
  27. data/lib/polyphony/extensions/socket.rb +2 -0
  28. data/lib/polyphony/extensions.rb +1 -0
  29. data/lib/polyphony/version.rb +1 -1
  30. data/polyphony.gemspec +2 -2
  31. data/test/test_backend.rb +5 -1
  32. data/test/test_enumerator.rb +46 -0
  33. data/test/test_global_api.rb +1 -1
  34. data/test/test_io.rb +241 -216
  35. data/test/test_socket.rb +1 -1
  36. data/test/test_thread_pool.rb +3 -3
  37. data/vendor/liburing/.github/workflows/build.yml +51 -5
  38. data/vendor/liburing/.github/workflows/shellcheck.yml +1 -1
  39. data/vendor/liburing/.gitignore +6 -123
  40. data/vendor/liburing/CHANGELOG +35 -0
  41. data/vendor/liburing/CITATION.cff +11 -0
  42. data/vendor/liburing/LICENSE +16 -3
  43. data/vendor/liburing/Makefile +3 -1
  44. data/vendor/liburing/Makefile.common +1 -0
  45. data/vendor/liburing/README +14 -2
  46. data/vendor/liburing/SECURITY.md +6 -0
  47. data/vendor/liburing/configure +16 -15
  48. data/vendor/liburing/examples/Makefile +4 -1
  49. data/vendor/liburing/examples/io_uring-udp.c +395 -0
  50. data/vendor/liburing/examples/poll-bench.c +101 -0
  51. data/vendor/liburing/examples/send-zerocopy.c +339 -0
  52. data/vendor/liburing/liburing.spec +1 -1
  53. data/vendor/liburing/man/io_uring.7 +38 -11
  54. data/vendor/liburing/man/io_uring_buf_ring_add.3 +53 -0
  55. data/vendor/liburing/man/io_uring_buf_ring_advance.3 +31 -0
  56. data/vendor/liburing/man/io_uring_buf_ring_cq_advance.3 +41 -0
  57. data/vendor/liburing/man/io_uring_buf_ring_init.3 +30 -0
  58. data/vendor/liburing/man/io_uring_buf_ring_mask.3 +27 -0
  59. data/vendor/liburing/man/io_uring_cq_advance.3 +29 -15
  60. data/vendor/liburing/man/io_uring_cq_has_overflow.3 +25 -0
  61. data/vendor/liburing/man/io_uring_cq_ready.3 +9 -8
  62. data/vendor/liburing/man/io_uring_cqe_get_data.3 +32 -13
  63. data/vendor/liburing/man/io_uring_cqe_get_data64.3 +1 -0
  64. data/vendor/liburing/man/io_uring_cqe_seen.3 +22 -12
  65. data/vendor/liburing/man/io_uring_enter.2 +249 -32
  66. data/vendor/liburing/man/io_uring_enter2.2 +1 -0
  67. data/vendor/liburing/man/io_uring_free_probe.3 +11 -8
  68. data/vendor/liburing/man/io_uring_get_events.3 +33 -0
  69. data/vendor/liburing/man/io_uring_get_probe.3 +9 -8
  70. data/vendor/liburing/man/io_uring_get_sqe.3 +29 -10
  71. data/vendor/liburing/man/io_uring_opcode_supported.3 +11 -10
  72. data/vendor/liburing/man/io_uring_peek_cqe.3 +38 -0
  73. data/vendor/liburing/man/io_uring_prep_accept.3 +197 -0
  74. data/vendor/liburing/man/io_uring_prep_accept_direct.3 +1 -0
  75. data/vendor/liburing/man/io_uring_prep_cancel.3 +118 -0
  76. data/vendor/liburing/man/io_uring_prep_cancel64.3 +1 -0
  77. data/vendor/liburing/man/io_uring_prep_close.3 +59 -0
  78. data/vendor/liburing/man/io_uring_prep_close_direct.3 +1 -0
  79. data/vendor/liburing/man/io_uring_prep_connect.3 +66 -0
  80. data/vendor/liburing/man/io_uring_prep_fadvise.3 +59 -0
  81. data/vendor/liburing/man/io_uring_prep_fallocate.3 +59 -0
  82. data/vendor/liburing/man/io_uring_prep_files_update.3 +92 -0
  83. data/vendor/liburing/man/io_uring_prep_fsync.3 +70 -0
  84. data/vendor/liburing/man/io_uring_prep_link.3 +1 -0
  85. data/vendor/liburing/man/io_uring_prep_linkat.3 +91 -0
  86. data/vendor/liburing/man/io_uring_prep_madvise.3 +56 -0
  87. data/vendor/liburing/man/io_uring_prep_mkdir.3 +1 -0
  88. data/vendor/liburing/man/io_uring_prep_mkdirat.3 +83 -0
  89. data/vendor/liburing/man/io_uring_prep_msg_ring.3 +39 -25
  90. data/vendor/liburing/man/io_uring_prep_multishot_accept.3 +1 -0
  91. data/vendor/liburing/man/io_uring_prep_multishot_accept_direct.3 +1 -0
  92. data/vendor/liburing/man/io_uring_prep_nop.3 +28 -0
  93. data/vendor/liburing/man/io_uring_prep_openat.3 +117 -0
  94. data/vendor/liburing/man/io_uring_prep_openat2.3 +117 -0
  95. data/vendor/liburing/man/io_uring_prep_openat2_direct.3 +1 -0
  96. data/vendor/liburing/man/io_uring_prep_openat_direct.3 +1 -0
  97. data/vendor/liburing/man/io_uring_prep_poll_add.3 +72 -0
  98. data/vendor/liburing/man/io_uring_prep_poll_multishot.3 +1 -0
  99. data/vendor/liburing/man/io_uring_prep_poll_remove.3 +55 -0
  100. data/vendor/liburing/man/io_uring_prep_poll_update.3 +89 -0
  101. data/vendor/liburing/man/io_uring_prep_provide_buffers.3 +131 -0
  102. data/vendor/liburing/man/io_uring_prep_read.3 +33 -14
  103. data/vendor/liburing/man/io_uring_prep_read_fixed.3 +39 -21
  104. data/vendor/liburing/man/io_uring_prep_readv.3 +49 -15
  105. data/vendor/liburing/man/io_uring_prep_readv2.3 +49 -17
  106. data/vendor/liburing/man/io_uring_prep_recv.3 +105 -0
  107. data/vendor/liburing/man/io_uring_prep_recv_multishot.3 +1 -0
  108. data/vendor/liburing/man/io_uring_prep_recvmsg.3 +124 -0
  109. data/vendor/liburing/man/io_uring_prep_recvmsg_multishot.3 +1 -0
  110. data/vendor/liburing/man/io_uring_prep_remove_buffers.3 +52 -0
  111. data/vendor/liburing/man/io_uring_prep_rename.3 +1 -0
  112. data/vendor/liburing/man/io_uring_prep_renameat.3 +96 -0
  113. data/vendor/liburing/man/io_uring_prep_send.3 +57 -0
  114. data/vendor/liburing/man/io_uring_prep_send_zc.3 +64 -0
  115. data/vendor/liburing/man/io_uring_prep_sendmsg.3 +69 -0
  116. data/vendor/liburing/man/io_uring_prep_shutdown.3 +53 -0
  117. data/vendor/liburing/man/io_uring_prep_socket.3 +118 -0
  118. data/vendor/liburing/man/io_uring_prep_socket_direct.3 +1 -0
  119. data/vendor/liburing/man/io_uring_prep_socket_direct_alloc.3 +1 -0
  120. data/vendor/liburing/man/io_uring_prep_splice.3 +80 -0
  121. data/vendor/liburing/man/io_uring_prep_statx.3 +74 -0
  122. data/vendor/liburing/man/io_uring_prep_symlink.3 +1 -0
  123. data/vendor/liburing/man/io_uring_prep_symlinkat.3 +85 -0
  124. data/vendor/liburing/man/io_uring_prep_sync_file_range.3 +59 -0
  125. data/vendor/liburing/man/io_uring_prep_tee.3 +74 -0
  126. data/vendor/liburing/man/io_uring_prep_timeout.3 +95 -0
  127. data/vendor/liburing/man/io_uring_prep_timeout_remove.3 +1 -0
  128. data/vendor/liburing/man/io_uring_prep_timeout_update.3 +98 -0
  129. data/vendor/liburing/man/io_uring_prep_unlink.3 +1 -0
  130. data/vendor/liburing/man/io_uring_prep_unlinkat.3 +82 -0
  131. data/vendor/liburing/man/io_uring_prep_write.3 +32 -15
  132. data/vendor/liburing/man/io_uring_prep_write_fixed.3 +39 -21
  133. data/vendor/liburing/man/io_uring_prep_writev.3 +50 -16
  134. data/vendor/liburing/man/io_uring_prep_writev2.3 +50 -17
  135. data/vendor/liburing/man/io_uring_queue_exit.3 +3 -4
  136. data/vendor/liburing/man/io_uring_queue_init.3 +58 -13
  137. data/vendor/liburing/man/io_uring_queue_init_params.3 +1 -0
  138. data/vendor/liburing/man/io_uring_recvmsg_cmsg_firsthdr.3 +1 -0
  139. data/vendor/liburing/man/io_uring_recvmsg_cmsg_nexthdr.3 +1 -0
  140. data/vendor/liburing/man/io_uring_recvmsg_name.3 +1 -0
  141. data/vendor/liburing/man/io_uring_recvmsg_out.3 +78 -0
  142. data/vendor/liburing/man/io_uring_recvmsg_payload.3 +1 -0
  143. data/vendor/liburing/man/io_uring_recvmsg_payload_length.3 +1 -0
  144. data/vendor/liburing/man/io_uring_recvmsg_validate.3 +1 -0
  145. data/vendor/liburing/man/io_uring_register.2 +153 -13
  146. data/vendor/liburing/man/io_uring_register_buf_ring.3 +140 -0
  147. data/vendor/liburing/man/io_uring_register_buffers.3 +32 -12
  148. data/vendor/liburing/man/io_uring_register_eventfd.3 +51 -0
  149. data/vendor/liburing/man/io_uring_register_eventfd_async.3 +1 -0
  150. data/vendor/liburing/man/io_uring_register_file_alloc_range.3 +52 -0
  151. data/vendor/liburing/man/io_uring_register_files.3 +33 -11
  152. data/vendor/liburing/man/io_uring_register_files_sparse.3 +1 -0
  153. data/vendor/liburing/man/io_uring_register_iowq_aff.3 +61 -0
  154. data/vendor/liburing/man/io_uring_register_iowq_max_workers.3 +71 -0
  155. data/vendor/liburing/man/io_uring_register_ring_fd.3 +49 -0
  156. data/vendor/liburing/man/io_uring_register_sync_cancel.3 +71 -0
  157. data/vendor/liburing/man/io_uring_setup.2 +119 -13
  158. data/vendor/liburing/man/io_uring_sq_ready.3 +14 -8
  159. data/vendor/liburing/man/io_uring_sq_space_left.3 +9 -9
  160. data/vendor/liburing/man/io_uring_sqe_set_data.3 +29 -11
  161. data/vendor/liburing/man/io_uring_sqe_set_data64.3 +1 -0
  162. data/vendor/liburing/man/io_uring_sqe_set_flags.3 +38 -11
  163. data/vendor/liburing/man/io_uring_sqring_wait.3 +13 -9
  164. data/vendor/liburing/man/io_uring_submit.3 +29 -12
  165. data/vendor/liburing/man/io_uring_submit_and_get_events.3 +31 -0
  166. data/vendor/liburing/man/io_uring_submit_and_wait.3 +16 -12
  167. data/vendor/liburing/man/io_uring_submit_and_wait_timeout.3 +30 -23
  168. data/vendor/liburing/man/io_uring_unregister_buf_ring.3 +30 -0
  169. data/vendor/liburing/man/io_uring_unregister_buffers.3 +11 -10
  170. data/vendor/liburing/man/io_uring_unregister_eventfd.3 +1 -0
  171. data/vendor/liburing/man/io_uring_unregister_files.3 +11 -10
  172. data/vendor/liburing/man/io_uring_unregister_iowq_aff.3 +1 -0
  173. data/vendor/liburing/man/io_uring_unregister_ring_fd.3 +32 -0
  174. data/vendor/liburing/man/io_uring_wait_cqe.3 +19 -12
  175. data/vendor/liburing/man/io_uring_wait_cqe_nr.3 +21 -14
  176. data/vendor/liburing/man/io_uring_wait_cqe_timeout.3 +27 -13
  177. data/vendor/liburing/man/io_uring_wait_cqes.3 +24 -14
  178. data/vendor/liburing/src/Makefile +8 -7
  179. data/vendor/liburing/src/arch/aarch64/lib.h +48 -0
  180. data/vendor/liburing/src/arch/aarch64/syscall.h +0 -4
  181. data/vendor/liburing/src/arch/generic/lib.h +0 -4
  182. data/vendor/liburing/src/arch/generic/syscall.h +29 -16
  183. data/vendor/liburing/src/arch/syscall-defs.h +41 -14
  184. data/vendor/liburing/src/arch/x86/lib.h +0 -21
  185. data/vendor/liburing/src/arch/x86/syscall.h +146 -10
  186. data/vendor/liburing/src/include/liburing/io_uring.h +245 -5
  187. data/vendor/liburing/src/include/liburing.h +468 -35
  188. data/vendor/liburing/src/int_flags.h +1 -0
  189. data/vendor/liburing/src/lib.h +20 -16
  190. data/vendor/liburing/src/liburing.map +16 -0
  191. data/vendor/liburing/src/nolibc.c +1 -1
  192. data/vendor/liburing/src/queue.c +87 -55
  193. data/vendor/liburing/src/register.c +129 -53
  194. data/vendor/liburing/src/setup.c +65 -28
  195. data/vendor/liburing/src/syscall.c +14 -32
  196. data/vendor/liburing/src/syscall.h +12 -64
  197. data/vendor/liburing/test/{232c93d07b74-test.c → 232c93d07b74.c} +8 -9
  198. data/vendor/liburing/test/{35fa71a030ca-test.c → 35fa71a030ca.c} +4 -4
  199. data/vendor/liburing/test/{500f9fbadef8-test.c → 500f9fbadef8.c} +7 -7
  200. data/vendor/liburing/test/{7ad0e4b2f83c-test.c → 7ad0e4b2f83c.c} +8 -7
  201. data/vendor/liburing/test/{8a9973408177-test.c → 8a9973408177.c} +4 -3
  202. data/vendor/liburing/test/{917257daa0fe-test.c → 917257daa0fe.c} +3 -2
  203. data/vendor/liburing/test/Makefile +60 -62
  204. data/vendor/liburing/test/{a0908ae19763-test.c → a0908ae19763.c} +3 -2
  205. data/vendor/liburing/test/{a4c0b3decb33-test.c → a4c0b3decb33.c} +3 -2
  206. data/vendor/liburing/test/accept-link.c +5 -4
  207. data/vendor/liburing/test/accept-reuse.c +17 -16
  208. data/vendor/liburing/test/accept-test.c +14 -10
  209. data/vendor/liburing/test/accept.c +529 -107
  210. data/vendor/liburing/test/across-fork.c +7 -6
  211. data/vendor/liburing/test/{b19062a56726-test.c → b19062a56726.c} +3 -2
  212. data/vendor/liburing/test/{b5837bd5311d-test.c → b5837bd5311d.c} +10 -9
  213. data/vendor/liburing/test/buf-ring.c +420 -0
  214. data/vendor/liburing/test/{ce593a6c480a-test.c → ce593a6c480a.c} +15 -12
  215. data/vendor/liburing/test/connect.c +8 -7
  216. data/vendor/liburing/test/cq-full.c +5 -4
  217. data/vendor/liburing/test/cq-overflow.c +242 -12
  218. data/vendor/liburing/test/cq-peek-batch.c +5 -4
  219. data/vendor/liburing/test/cq-ready.c +5 -4
  220. data/vendor/liburing/test/cq-size.c +5 -4
  221. data/vendor/liburing/test/{d4ae271dfaae-test.c → d4ae271dfaae.c} +2 -2
  222. data/vendor/liburing/test/{d77a67ed5f27-test.c → d77a67ed5f27.c} +6 -6
  223. data/vendor/liburing/test/defer-taskrun.c +336 -0
  224. data/vendor/liburing/test/defer.c +26 -14
  225. data/vendor/liburing/test/double-poll-crash.c +15 -5
  226. data/vendor/liburing/test/drop-submit.c +5 -3
  227. data/vendor/liburing/test/{eeed8b54e0df-test.c → eeed8b54e0df.c} +7 -6
  228. data/vendor/liburing/test/empty-eownerdead.c +4 -4
  229. data/vendor/liburing/test/eventfd-disable.c +48 -20
  230. data/vendor/liburing/test/eventfd-reg.c +10 -9
  231. data/vendor/liburing/test/eventfd-ring.c +13 -12
  232. data/vendor/liburing/test/eventfd.c +13 -12
  233. data/vendor/liburing/test/exit-no-cleanup.c +1 -1
  234. data/vendor/liburing/test/fadvise.c +3 -3
  235. data/vendor/liburing/test/fallocate.c +16 -9
  236. data/vendor/liburing/test/{fc2a85cb02ef-test.c → fc2a85cb02ef.c} +4 -3
  237. data/vendor/liburing/test/fd-pass.c +187 -0
  238. data/vendor/liburing/test/file-register.c +302 -36
  239. data/vendor/liburing/test/file-update.c +62 -4
  240. data/vendor/liburing/test/file-verify.c +6 -2
  241. data/vendor/liburing/test/files-exit-hang-poll.c +11 -25
  242. data/vendor/liburing/test/files-exit-hang-timeout.c +13 -10
  243. data/vendor/liburing/test/fixed-buf-iter.c +115 -0
  244. data/vendor/liburing/test/fixed-link.c +10 -10
  245. data/vendor/liburing/test/fixed-reuse.c +160 -0
  246. data/vendor/liburing/test/fpos.c +6 -3
  247. data/vendor/liburing/test/fsync.c +3 -3
  248. data/vendor/liburing/test/hardlink.c +10 -6
  249. data/vendor/liburing/test/helpers.c +137 -4
  250. data/vendor/liburing/test/helpers.h +27 -0
  251. data/vendor/liburing/test/io-cancel.c +16 -11
  252. data/vendor/liburing/test/io_uring_enter.c +46 -81
  253. data/vendor/liburing/test/io_uring_passthrough.c +451 -0
  254. data/vendor/liburing/test/io_uring_register.c +59 -229
  255. data/vendor/liburing/test/io_uring_setup.c +24 -29
  256. data/vendor/liburing/test/iopoll-leak.c +85 -0
  257. data/vendor/liburing/test/iopoll.c +16 -9
  258. data/vendor/liburing/test/lfs-openat-write.c +3 -1
  259. data/vendor/liburing/test/link-timeout.c +4 -3
  260. data/vendor/liburing/test/link.c +8 -7
  261. data/vendor/liburing/test/madvise.c +2 -2
  262. data/vendor/liburing/test/mkdir.c +9 -5
  263. data/vendor/liburing/test/msg-ring.c +46 -20
  264. data/vendor/liburing/test/multicqes_drain.c +51 -12
  265. data/vendor/liburing/test/nolibc.c +60 -0
  266. data/vendor/liburing/test/nop.c +78 -16
  267. data/vendor/liburing/test/nvme.h +168 -0
  268. data/vendor/liburing/test/open-direct-link.c +188 -0
  269. data/vendor/liburing/test/open-direct-pick.c +180 -0
  270. data/vendor/liburing/test/openat2.c +3 -3
  271. data/vendor/liburing/test/poll-cancel-all.c +472 -0
  272. data/vendor/liburing/test/poll-link.c +9 -18
  273. data/vendor/liburing/test/poll-mshot-overflow.c +162 -0
  274. data/vendor/liburing/test/poll-mshot-update.c +83 -33
  275. data/vendor/liburing/test/pollfree.c +2 -2
  276. data/vendor/liburing/test/read-before-exit.c +112 -0
  277. data/vendor/liburing/test/read-write.c +83 -1
  278. data/vendor/liburing/test/recv-msgall-stream.c +398 -0
  279. data/vendor/liburing/test/recv-msgall.c +265 -0
  280. data/vendor/liburing/test/recv-multishot.c +505 -0
  281. data/vendor/liburing/test/rename.c +2 -5
  282. data/vendor/liburing/test/ring-leak.c +97 -0
  283. data/vendor/liburing/test/ringbuf-read.c +200 -0
  284. data/vendor/liburing/test/rsrc_tags.c +25 -13
  285. data/vendor/liburing/test/runtests-quiet.sh +11 -0
  286. data/vendor/liburing/test/runtests.sh +18 -20
  287. data/vendor/liburing/test/rw_merge_test.c +3 -2
  288. data/vendor/liburing/test/send-zerocopy.c +684 -0
  289. data/vendor/liburing/test/send_recv.c +49 -2
  290. data/vendor/liburing/test/send_recvmsg.c +165 -55
  291. data/vendor/liburing/test/shutdown.c +3 -4
  292. data/vendor/liburing/test/sigfd-deadlock.c +22 -8
  293. data/vendor/liburing/test/single-issuer.c +171 -0
  294. data/vendor/liburing/test/socket-rw-eagain.c +2 -12
  295. data/vendor/liburing/test/socket-rw-offset.c +2 -11
  296. data/vendor/liburing/test/socket-rw.c +2 -11
  297. data/vendor/liburing/test/socket.c +409 -0
  298. data/vendor/liburing/test/sq-poll-dup.c +1 -1
  299. data/vendor/liburing/test/sq-poll-share.c +1 -1
  300. data/vendor/liburing/test/statx.c +2 -2
  301. data/vendor/liburing/test/submit-and-wait.c +108 -0
  302. data/vendor/liburing/test/submit-link-fail.c +5 -3
  303. data/vendor/liburing/test/submit-reuse.c +0 -2
  304. data/vendor/liburing/test/sync-cancel.c +235 -0
  305. data/vendor/liburing/test/test.h +35 -0
  306. data/vendor/liburing/test/timeout-overflow.c +11 -11
  307. data/vendor/liburing/test/timeout.c +7 -7
  308. data/vendor/liburing/test/tty-write-dpoll.c +60 -0
  309. data/vendor/liburing/test/unlink.c +1 -1
  310. data/vendor/liburing/test/xattr.c +425 -0
  311. metadata +148 -26
  312. data/Gemfile.lock +0 -82
@@ -8,7 +8,6 @@
8
8
  #include "liburing/compat.h"
9
9
  #include "liburing/io_uring.h"
10
10
 
11
-
12
11
  int io_uring_register_buffers_update_tag(struct io_uring *ring, unsigned off,
13
12
  const struct iovec *iovecs,
14
13
  const __u64 *tags,
@@ -21,8 +20,7 @@ int io_uring_register_buffers_update_tag(struct io_uring *ring, unsigned off,
21
20
  .nr = nr,
22
21
  };
23
22
 
24
- return ____sys_io_uring_register(ring->ring_fd,
25
- IORING_REGISTER_BUFFERS_UPDATE, &up,
23
+ return __sys_io_uring_register(ring->ring_fd,IORING_REGISTER_BUFFERS_UPDATE, &up,
26
24
  sizeof(up));
27
25
  }
28
26
 
@@ -37,8 +35,20 @@ int io_uring_register_buffers_tags(struct io_uring *ring,
37
35
  .tags = (unsigned long)tags,
38
36
  };
39
37
 
40
- return ____sys_io_uring_register(ring->ring_fd, IORING_REGISTER_BUFFERS2,
41
- &reg, sizeof(reg));
38
+ return __sys_io_uring_register(ring->ring_fd,
39
+ IORING_REGISTER_BUFFERS2, &reg,
40
+ sizeof(reg));
41
+ }
42
+
43
+ int io_uring_register_buffers_sparse(struct io_uring *ring, unsigned nr)
44
+ {
45
+ struct io_uring_rsrc_register reg = {
46
+ .flags = IORING_RSRC_REGISTER_SPARSE,
47
+ .nr = nr,
48
+ };
49
+
50
+ return __sys_io_uring_register(ring->ring_fd, IORING_REGISTER_BUFFERS2,
51
+ &reg, sizeof(reg));
42
52
  }
43
53
 
44
54
  int io_uring_register_buffers(struct io_uring *ring, const struct iovec *iovecs,
@@ -46,8 +56,8 @@ int io_uring_register_buffers(struct io_uring *ring, const struct iovec *iovecs,
46
56
  {
47
57
  int ret;
48
58
 
49
- ret = ____sys_io_uring_register(ring->ring_fd, IORING_REGISTER_BUFFERS,
50
- iovecs, nr_iovecs);
59
+ ret = __sys_io_uring_register(ring->ring_fd, IORING_REGISTER_BUFFERS,
60
+ iovecs, nr_iovecs);
51
61
  return (ret < 0) ? ret : 0;
52
62
  }
53
63
 
@@ -55,8 +65,8 @@ int io_uring_unregister_buffers(struct io_uring *ring)
55
65
  {
56
66
  int ret;
57
67
 
58
- ret = ____sys_io_uring_register(ring->ring_fd, IORING_UNREGISTER_BUFFERS,
59
- NULL, 0);
68
+ ret = __sys_io_uring_register(ring->ring_fd, IORING_UNREGISTER_BUFFERS,
69
+ NULL, 0);
60
70
  return (ret < 0) ? ret : 0;
61
71
  }
62
72
 
@@ -71,9 +81,9 @@ int io_uring_register_files_update_tag(struct io_uring *ring, unsigned off,
71
81
  .nr = nr_files,
72
82
  };
73
83
 
74
- return ____sys_io_uring_register(ring->ring_fd,
75
- IORING_REGISTER_FILES_UPDATE2, &up,
76
- sizeof(up));
84
+ return __sys_io_uring_register(ring->ring_fd,
85
+ IORING_REGISTER_FILES_UPDATE2, &up,
86
+ sizeof(up));
77
87
  }
78
88
 
79
89
  /*
@@ -84,16 +94,16 @@ int io_uring_register_files_update_tag(struct io_uring *ring, unsigned off,
84
94
  * Returns number of files updated on success, -ERROR on failure.
85
95
  */
86
96
  int io_uring_register_files_update(struct io_uring *ring, unsigned off,
87
- int *files, unsigned nr_files)
97
+ const int *files, unsigned nr_files)
88
98
  {
89
99
  struct io_uring_files_update up = {
90
100
  .offset = off,
91
101
  .fds = (unsigned long) files,
92
102
  };
93
103
 
94
- return ____sys_io_uring_register(ring->ring_fd,
95
- IORING_REGISTER_FILES_UPDATE, &up,
96
- nr_files);
104
+ return __sys_io_uring_register(ring->ring_fd,
105
+ IORING_REGISTER_FILES_UPDATE, &up,
106
+ nr_files);
97
107
  }
98
108
 
99
109
  static int increase_rlimit_nofile(unsigned nr)
@@ -113,6 +123,31 @@ static int increase_rlimit_nofile(unsigned nr)
113
123
  return 0;
114
124
  }
115
125
 
126
+ int io_uring_register_files_sparse(struct io_uring *ring, unsigned nr)
127
+ {
128
+ struct io_uring_rsrc_register reg = {
129
+ .flags = IORING_RSRC_REGISTER_SPARSE,
130
+ .nr = nr,
131
+ };
132
+ int ret, did_increase = 0;
133
+
134
+ do {
135
+ ret = __sys_io_uring_register(ring->ring_fd,
136
+ IORING_REGISTER_FILES2, &reg,
137
+ sizeof(reg));
138
+ if (ret >= 0)
139
+ break;
140
+ if (ret == -EMFILE && !did_increase) {
141
+ did_increase = 1;
142
+ increase_rlimit_nofile(nr);
143
+ continue;
144
+ }
145
+ break;
146
+ } while (1);
147
+
148
+ return ret;
149
+ }
150
+
116
151
  int io_uring_register_files_tags(struct io_uring *ring, const int *files,
117
152
  const __u64 *tags, unsigned nr)
118
153
  {
@@ -124,9 +159,9 @@ int io_uring_register_files_tags(struct io_uring *ring, const int *files,
124
159
  int ret, did_increase = 0;
125
160
 
126
161
  do {
127
- ret = ____sys_io_uring_register(ring->ring_fd,
128
- IORING_REGISTER_FILES2, &reg,
129
- sizeof(reg));
162
+ ret = __sys_io_uring_register(ring->ring_fd,
163
+ IORING_REGISTER_FILES2, &reg,
164
+ sizeof(reg));
130
165
  if (ret >= 0)
131
166
  break;
132
167
  if (ret == -EMFILE && !did_increase) {
@@ -146,9 +181,9 @@ int io_uring_register_files(struct io_uring *ring, const int *files,
146
181
  int ret, did_increase = 0;
147
182
 
148
183
  do {
149
- ret = ____sys_io_uring_register(ring->ring_fd,
150
- IORING_REGISTER_FILES, files,
151
- nr_files);
184
+ ret = __sys_io_uring_register(ring->ring_fd,
185
+ IORING_REGISTER_FILES, files,
186
+ nr_files);
152
187
  if (ret >= 0)
153
188
  break;
154
189
  if (ret == -EMFILE && !did_increase) {
@@ -166,8 +201,8 @@ int io_uring_unregister_files(struct io_uring *ring)
166
201
  {
167
202
  int ret;
168
203
 
169
- ret = ____sys_io_uring_register(ring->ring_fd, IORING_UNREGISTER_FILES,
170
- NULL, 0);
204
+ ret = __sys_io_uring_register(ring->ring_fd, IORING_UNREGISTER_FILES,
205
+ NULL, 0);
171
206
  return (ret < 0) ? ret : 0;
172
207
  }
173
208
 
@@ -175,8 +210,8 @@ int io_uring_register_eventfd(struct io_uring *ring, int event_fd)
175
210
  {
176
211
  int ret;
177
212
 
178
- ret = ____sys_io_uring_register(ring->ring_fd, IORING_REGISTER_EVENTFD,
179
- &event_fd, 1);
213
+ ret = __sys_io_uring_register(ring->ring_fd, IORING_REGISTER_EVENTFD,
214
+ &event_fd, 1);
180
215
  return (ret < 0) ? ret : 0;
181
216
  }
182
217
 
@@ -184,8 +219,8 @@ int io_uring_unregister_eventfd(struct io_uring *ring)
184
219
  {
185
220
  int ret;
186
221
 
187
- ret = ____sys_io_uring_register(ring->ring_fd, IORING_UNREGISTER_EVENTFD,
188
- NULL, 0);
222
+ ret = __sys_io_uring_register(ring->ring_fd, IORING_UNREGISTER_EVENTFD,
223
+ NULL, 0);
189
224
  return (ret < 0) ? ret : 0;
190
225
  }
191
226
 
@@ -193,9 +228,9 @@ int io_uring_register_eventfd_async(struct io_uring *ring, int event_fd)
193
228
  {
194
229
  int ret;
195
230
 
196
- ret = ____sys_io_uring_register(ring->ring_fd,
197
- IORING_REGISTER_EVENTFD_ASYNC,
198
- &event_fd, 1);
231
+ ret = __sys_io_uring_register(ring->ring_fd,
232
+ IORING_REGISTER_EVENTFD_ASYNC, &event_fd,
233
+ 1);
199
234
  return (ret < 0) ? ret : 0;
200
235
  }
201
236
 
@@ -204,21 +239,21 @@ int io_uring_register_probe(struct io_uring *ring, struct io_uring_probe *p,
204
239
  {
205
240
  int ret;
206
241
 
207
- ret = ____sys_io_uring_register(ring->ring_fd, IORING_REGISTER_PROBE, p,
208
- nr_ops);
242
+ ret = __sys_io_uring_register(ring->ring_fd, IORING_REGISTER_PROBE, p,
243
+ nr_ops);
209
244
  return (ret < 0) ? ret : 0;
210
245
  }
211
246
 
212
247
  int io_uring_register_personality(struct io_uring *ring)
213
248
  {
214
- return ____sys_io_uring_register(ring->ring_fd,
215
- IORING_REGISTER_PERSONALITY, NULL, 0);
249
+ return __sys_io_uring_register(ring->ring_fd,
250
+ IORING_REGISTER_PERSONALITY, NULL, 0);
216
251
  }
217
252
 
218
253
  int io_uring_unregister_personality(struct io_uring *ring, int id)
219
254
  {
220
- return ____sys_io_uring_register(ring->ring_fd,
221
- IORING_UNREGISTER_PERSONALITY, NULL, id);
255
+ return __sys_io_uring_register(ring->ring_fd,
256
+ IORING_UNREGISTER_PERSONALITY, NULL, id);
222
257
  }
223
258
 
224
259
  int io_uring_register_restrictions(struct io_uring *ring,
@@ -227,36 +262,39 @@ int io_uring_register_restrictions(struct io_uring *ring,
227
262
  {
228
263
  int ret;
229
264
 
230
- ret = ____sys_io_uring_register(ring->ring_fd,
231
- IORING_REGISTER_RESTRICTIONS, res,
232
- nr_res);
265
+ ret = __sys_io_uring_register(ring->ring_fd,
266
+ IORING_REGISTER_RESTRICTIONS, res,
267
+ nr_res);
233
268
  return (ret < 0) ? ret : 0;
234
269
  }
235
270
 
236
271
  int io_uring_enable_rings(struct io_uring *ring)
237
272
  {
238
- return ____sys_io_uring_register(ring->ring_fd,
239
- IORING_REGISTER_ENABLE_RINGS, NULL, 0);
273
+ return __sys_io_uring_register(ring->ring_fd,
274
+ IORING_REGISTER_ENABLE_RINGS, NULL, 0);
240
275
  }
241
276
 
242
277
  int io_uring_register_iowq_aff(struct io_uring *ring, size_t cpusz,
243
278
  const cpu_set_t *mask)
244
279
  {
245
- return ____sys_io_uring_register(ring->ring_fd, IORING_REGISTER_IOWQ_AFF,
246
- mask, cpusz);
280
+ if (cpusz >= (1U << 31))
281
+ return -EINVAL;
282
+
283
+ return __sys_io_uring_register(ring->ring_fd, IORING_REGISTER_IOWQ_AFF,
284
+ mask, (int) cpusz);
247
285
  }
248
286
 
249
287
  int io_uring_unregister_iowq_aff(struct io_uring *ring)
250
288
  {
251
- return ____sys_io_uring_register(ring->ring_fd,
252
- IORING_UNREGISTER_IOWQ_AFF, NULL, 0);
289
+ return __sys_io_uring_register(ring->ring_fd,
290
+ IORING_UNREGISTER_IOWQ_AFF, NULL, 0);
253
291
  }
254
292
 
255
293
  int io_uring_register_iowq_max_workers(struct io_uring *ring, unsigned int *val)
256
294
  {
257
- return ____sys_io_uring_register(ring->ring_fd,
258
- IORING_REGISTER_IOWQ_MAX_WORKERS, val,
259
- 2);
295
+ return __sys_io_uring_register(ring->ring_fd,
296
+ IORING_REGISTER_IOWQ_MAX_WORKERS, val,
297
+ 2);
260
298
  }
261
299
 
262
300
  int io_uring_register_ring_fd(struct io_uring *ring)
@@ -267,8 +305,8 @@ int io_uring_register_ring_fd(struct io_uring *ring)
267
305
  };
268
306
  int ret;
269
307
 
270
- ret = ____sys_io_uring_register(ring->ring_fd, IORING_REGISTER_RING_FDS,
271
- &up, 1);
308
+ ret = __sys_io_uring_register(ring->ring_fd, IORING_REGISTER_RING_FDS,
309
+ &up, 1);
272
310
  if (ret == 1) {
273
311
  ring->enter_ring_fd = up.offset;
274
312
  ring->int_flags |= INT_FLAG_REG_RING;
@@ -276,6 +314,7 @@ int io_uring_register_ring_fd(struct io_uring *ring)
276
314
  return ret;
277
315
  }
278
316
 
317
+
279
318
  int io_uring_unregister_ring_fd(struct io_uring *ring)
280
319
  {
281
320
  struct io_uring_rsrc_update up = {
@@ -283,11 +322,48 @@ int io_uring_unregister_ring_fd(struct io_uring *ring)
283
322
  };
284
323
  int ret;
285
324
 
286
- ret = ____sys_io_uring_register(ring->ring_fd,
287
- IORING_UNREGISTER_RING_FDS, &up, 1);
325
+ ret = __sys_io_uring_register(ring->ring_fd, IORING_UNREGISTER_RING_FDS,
326
+ &up, 1);
288
327
  if (ret == 1) {
289
328
  ring->enter_ring_fd = ring->ring_fd;
290
329
  ring->int_flags &= ~INT_FLAG_REG_RING;
291
330
  }
292
331
  return ret;
293
332
  }
333
+
334
+ int io_uring_register_buf_ring(struct io_uring *ring,
335
+ struct io_uring_buf_reg *reg,
336
+ unsigned int __maybe_unused flags)
337
+ {
338
+ return __sys_io_uring_register(ring->ring_fd, IORING_REGISTER_PBUF_RING,
339
+ reg, 1);
340
+ }
341
+
342
+ int io_uring_unregister_buf_ring(struct io_uring *ring, int bgid)
343
+ {
344
+ struct io_uring_buf_reg reg = { .bgid = bgid };
345
+
346
+ return __sys_io_uring_register(ring->ring_fd,
347
+ IORING_UNREGISTER_PBUF_RING, &reg, 1);
348
+ }
349
+
350
+ int io_uring_register_sync_cancel(struct io_uring *ring,
351
+ struct io_uring_sync_cancel_reg *reg)
352
+ {
353
+ return __sys_io_uring_register(ring->ring_fd,
354
+ IORING_REGISTER_SYNC_CANCEL, reg, 1);
355
+ }
356
+
357
+ int io_uring_register_file_alloc_range(struct io_uring *ring,
358
+ unsigned off, unsigned len)
359
+ {
360
+ struct io_uring_file_index_range range;
361
+
362
+ memset(&range, 0, sizeof(range));
363
+ range.off = off;
364
+ range.len = len;
365
+
366
+ return __sys_io_uring_register(ring->ring_fd,
367
+ IORING_REGISTER_FILE_ALLOC_RANGE, &range,
368
+ 0);
369
+ }
@@ -8,7 +8,6 @@
8
8
  #include "liburing/compat.h"
9
9
  #include "liburing/io_uring.h"
10
10
 
11
-
12
11
  static void io_uring_unmap_rings(struct io_uring_sq *sq, struct io_uring_cq *cq)
13
12
  {
14
13
  __sys_munmap(sq->ring_ptr, sq->ring_sz);
@@ -22,8 +21,12 @@ static int io_uring_mmap(int fd, struct io_uring_params *p,
22
21
  size_t size;
23
22
  int ret;
24
23
 
24
+ size = sizeof(struct io_uring_cqe);
25
+ if (p->flags & IORING_SETUP_CQE32)
26
+ size += sizeof(struct io_uring_cqe);
27
+
25
28
  sq->ring_sz = p->sq_off.array + p->sq_entries * sizeof(unsigned);
26
- cq->ring_sz = p->cq_off.cqes + p->cq_entries * sizeof(struct io_uring_cqe);
29
+ cq->ring_sz = p->cq_off.cqes + p->cq_entries * size;
27
30
 
28
31
  if (p->features & IORING_FEAT_SINGLE_MMAP) {
29
32
  if (cq->ring_sz > sq->ring_sz)
@@ -57,8 +60,10 @@ static int io_uring_mmap(int fd, struct io_uring_params *p,
57
60
  sq->kdropped = sq->ring_ptr + p->sq_off.dropped;
58
61
  sq->array = sq->ring_ptr + p->sq_off.array;
59
62
 
60
- size = p->sq_entries * sizeof(struct io_uring_sqe);
61
- sq->sqes = __sys_mmap(0, size, PROT_READ | PROT_WRITE,
63
+ size = sizeof(struct io_uring_sqe);
64
+ if (p->flags & IORING_SETUP_SQE128)
65
+ size += 64;
66
+ sq->sqes = __sys_mmap(0, size * p->sq_entries, PROT_READ | PROT_WRITE,
62
67
  MAP_SHARED | MAP_POPULATE, fd, IORING_OFF_SQES);
63
68
  if (IS_ERR(sq->sqes)) {
64
69
  ret = PTR_ERR(sq->sqes);
@@ -75,6 +80,11 @@ err:
75
80
  cq->cqes = cq->ring_ptr + p->cq_off.cqes;
76
81
  if (p->cq_off.flags)
77
82
  cq->kflags = cq->ring_ptr + p->cq_off.flags;
83
+
84
+ sq->ring_mask = *sq->kring_mask;
85
+ sq->ring_entries = *sq->kring_entries;
86
+ cq->ring_mask = *cq->kring_mask;
87
+ cq->ring_entries = *cq->kring_entries;
78
88
  return 0;
79
89
  }
80
90
 
@@ -84,7 +94,8 @@ err:
84
94
  * Returns -errno on error, or zero on success. On success, 'ring'
85
95
  * contains the necessary information to read/write to the rings.
86
96
  */
87
- int io_uring_queue_mmap(int fd, struct io_uring_params *p, struct io_uring *ring)
97
+ __cold int io_uring_queue_mmap(int fd, struct io_uring_params *p,
98
+ struct io_uring *ring)
88
99
  {
89
100
  int ret;
90
101
 
@@ -102,7 +113,7 @@ int io_uring_queue_mmap(int fd, struct io_uring_params *p, struct io_uring *ring
102
113
  * Ensure that the mmap'ed rings aren't available to a child after a fork(2).
103
114
  * This uses madvise(..., MADV_DONTFORK) on the mmap'ed ranges.
104
115
  */
105
- int io_uring_ring_dontfork(struct io_uring *ring)
116
+ __cold int io_uring_ring_dontfork(struct io_uring *ring)
106
117
  {
107
118
  size_t len;
108
119
  int ret;
@@ -110,7 +121,10 @@ int io_uring_ring_dontfork(struct io_uring *ring)
110
121
  if (!ring->sq.ring_ptr || !ring->sq.sqes || !ring->cq.ring_ptr)
111
122
  return -EINVAL;
112
123
 
113
- len = *ring->sq.kring_entries * sizeof(struct io_uring_sqe);
124
+ len = sizeof(struct io_uring_sqe);
125
+ if (ring->flags & IORING_SETUP_SQE128)
126
+ len += 64;
127
+ len *= ring->sq.ring_entries;
114
128
  ret = __sys_madvise(ring->sq.sqes, len, MADV_DONTFORK);
115
129
  if (ret < 0)
116
130
  return ret;
@@ -130,12 +144,14 @@ int io_uring_ring_dontfork(struct io_uring *ring)
130
144
  return 0;
131
145
  }
132
146
 
133
- int io_uring_queue_init_params(unsigned entries, struct io_uring *ring,
134
- struct io_uring_params *p)
147
+ __cold int io_uring_queue_init_params(unsigned entries, struct io_uring *ring,
148
+ struct io_uring_params *p)
135
149
  {
136
150
  int fd, ret;
151
+ unsigned *sq_array;
152
+ unsigned sq_entries, index;
137
153
 
138
- fd = ____sys_io_uring_setup(entries, p);
154
+ fd = __sys_io_uring_setup(entries, p);
139
155
  if (fd < 0)
140
156
  return fd;
141
157
 
@@ -145,6 +161,14 @@ int io_uring_queue_init_params(unsigned entries, struct io_uring *ring,
145
161
  return ret;
146
162
  }
147
163
 
164
+ /*
165
+ * Directly map SQ slots to SQEs
166
+ */
167
+ sq_array = ring->sq.array;
168
+ sq_entries = ring->sq.ring_entries;
169
+ for (index = 0; index < sq_entries; index++)
170
+ sq_array[index] = index;
171
+
148
172
  ring->features = p->features;
149
173
  return 0;
150
174
  }
@@ -153,7 +177,8 @@ int io_uring_queue_init_params(unsigned entries, struct io_uring *ring,
153
177
  * Returns -errno on error, or zero on success. On success, 'ring'
154
178
  * contains the necessary information to read/write to the rings.
155
179
  */
156
- int io_uring_queue_init(unsigned entries, struct io_uring *ring, unsigned flags)
180
+ __cold int io_uring_queue_init(unsigned entries, struct io_uring *ring,
181
+ unsigned flags)
157
182
  {
158
183
  struct io_uring_params p;
159
184
 
@@ -163,12 +188,16 @@ int io_uring_queue_init(unsigned entries, struct io_uring *ring, unsigned flags)
163
188
  return io_uring_queue_init_params(entries, ring, &p);
164
189
  }
165
190
 
166
- void io_uring_queue_exit(struct io_uring *ring)
191
+ __cold void io_uring_queue_exit(struct io_uring *ring)
167
192
  {
168
193
  struct io_uring_sq *sq = &ring->sq;
169
194
  struct io_uring_cq *cq = &ring->cq;
195
+ size_t sqe_size;
170
196
 
171
- __sys_munmap(sq->sqes, *sq->kring_entries * sizeof(struct io_uring_sqe));
197
+ sqe_size = sizeof(struct io_uring_sqe);
198
+ if (ring->flags & IORING_SETUP_SQE128)
199
+ sqe_size += 64;
200
+ __sys_munmap(sq->sqes, sqe_size * sq->ring_entries);
172
201
  io_uring_unmap_rings(sq, cq);
173
202
  /*
174
203
  * Not strictly required, but frees up the slot we used now rather
@@ -179,7 +208,7 @@ void io_uring_queue_exit(struct io_uring *ring)
179
208
  __sys_close(ring->ring_fd);
180
209
  }
181
210
 
182
- struct io_uring_probe *io_uring_get_probe_ring(struct io_uring *ring)
211
+ __cold struct io_uring_probe *io_uring_get_probe_ring(struct io_uring *ring)
183
212
  {
184
213
  struct io_uring_probe *probe;
185
214
  size_t len;
@@ -199,7 +228,7 @@ struct io_uring_probe *io_uring_get_probe_ring(struct io_uring *ring)
199
228
  return NULL;
200
229
  }
201
230
 
202
- struct io_uring_probe *io_uring_get_probe(void)
231
+ __cold struct io_uring_probe *io_uring_get_probe(void)
203
232
  {
204
233
  struct io_uring ring;
205
234
  struct io_uring_probe *probe;
@@ -214,42 +243,49 @@ struct io_uring_probe *io_uring_get_probe(void)
214
243
  return probe;
215
244
  }
216
245
 
217
- void io_uring_free_probe(struct io_uring_probe *probe)
246
+ __cold void io_uring_free_probe(struct io_uring_probe *probe)
218
247
  {
219
248
  uring_free(probe);
220
249
  }
221
250
 
222
- static inline int __fls(int x)
251
+ static inline int __fls(unsigned long x)
223
252
  {
224
253
  if (!x)
225
254
  return 0;
226
- return 8 * sizeof(x) - __builtin_clz(x);
255
+ return 8 * sizeof(x) - __builtin_clzl(x);
227
256
  }
228
257
 
229
258
  static unsigned roundup_pow2(unsigned depth)
230
259
  {
231
- return 1UL << __fls(depth - 1);
260
+ return 1U << __fls(depth - 1);
232
261
  }
233
262
 
234
- static size_t npages(size_t size, unsigned page_size)
263
+ static size_t npages(size_t size, long page_size)
235
264
  {
236
265
  size--;
237
266
  size /= page_size;
238
- return __fls(size);
267
+ return __fls((int) size);
239
268
  }
240
269
 
241
270
  #define KRING_SIZE 320
242
271
 
243
- static size_t rings_size(unsigned entries, unsigned cq_entries, unsigned page_size)
272
+ static size_t rings_size(struct io_uring_params *p, unsigned entries,
273
+ unsigned cq_entries, long page_size)
244
274
  {
245
275
  size_t pages, sq_size, cq_size;
246
276
 
247
- cq_size = KRING_SIZE;
248
- cq_size += cq_entries * sizeof(struct io_uring_cqe);
277
+ cq_size = sizeof(struct io_uring_cqe);
278
+ if (p->flags & IORING_SETUP_CQE32)
279
+ cq_size += sizeof(struct io_uring_cqe);
280
+ cq_size *= cq_entries;
281
+ cq_size += KRING_SIZE;
249
282
  cq_size = (cq_size + 63) & ~63UL;
250
283
  pages = (size_t) 1 << npages(cq_size, page_size);
251
284
 
252
- sq_size = sizeof(struct io_uring_sqe) * entries;
285
+ sq_size = sizeof(struct io_uring_sqe);
286
+ if (p->flags & IORING_SETUP_SQE128)
287
+ sq_size += 64;
288
+ sq_size *= entries;
253
289
  pages += (size_t) 1 << npages(sq_size, page_size);
254
290
  return pages * page_size;
255
291
  }
@@ -265,7 +301,8 @@ static size_t rings_size(unsigned entries, unsigned cq_entries, unsigned page_si
265
301
  * return the required memory so that the caller can ensure that enough space
266
302
  * is available before setting up a ring with the specified parameters.
267
303
  */
268
- ssize_t io_uring_mlock_size_params(unsigned entries, struct io_uring_params *p)
304
+ __cold ssize_t io_uring_mlock_size_params(unsigned entries,
305
+ struct io_uring_params *p)
269
306
  {
270
307
  struct io_uring_params lp = { };
271
308
  struct io_uring ring;
@@ -317,14 +354,14 @@ ssize_t io_uring_mlock_size_params(unsigned entries, struct io_uring_params *p)
317
354
  }
318
355
 
319
356
  page_size = get_page_size();
320
- return rings_size(entries, cq_entries, page_size);
357
+ return rings_size(p, entries, cq_entries, page_size);
321
358
  }
322
359
 
323
360
  /*
324
361
  * Return required ulimit -l memory space for a given ring setup. See
325
362
  * @io_uring_mlock_size_params().
326
363
  */
327
- ssize_t io_uring_mlock_size(unsigned entries, unsigned flags)
364
+ __cold ssize_t io_uring_mlock_size(unsigned entries, unsigned flags)
328
365
  {
329
366
  struct io_uring_params p = { .flags = flags, };
330
367
 
@@ -1,47 +1,29 @@
1
1
  /* SPDX-License-Identifier: MIT */
2
- #define _DEFAULT_SOURCE
3
2
 
4
- /*
5
- * Functions in this file require libc, only build them when we use libc.
6
- *
7
- * Note:
8
- * liburing's tests still need these functions.
9
- */
10
- #if defined(CONFIG_NOLIBC) && !defined(LIBURING_BUILD_TEST)
11
- # error "This file should only be compiled for libc build, or for liburing tests"
12
- #endif
13
-
14
- /*
15
- * Will go away once libc support is there
16
- */
17
- #include <unistd.h>
18
- #include <sys/syscall.h>
19
- #include <sys/uio.h>
20
- #include "liburing/compat.h"
21
- #include "liburing/io_uring.h"
22
3
  #include "syscall.h"
4
+ #include <liburing.h>
23
5
 
24
- int __sys_io_uring_register(int fd, unsigned opcode, const void *arg,
25
- unsigned nr_args)
6
+ int io_uring_enter(unsigned int fd, unsigned int to_submit,
7
+ unsigned int min_complete, unsigned int flags, sigset_t *sig)
26
8
  {
27
- return syscall(__NR_io_uring_register, fd, opcode, arg, nr_args);
9
+ return __sys_io_uring_enter(fd, to_submit, min_complete, flags, sig);
28
10
  }
29
11
 
30
- int __sys_io_uring_setup(unsigned entries, struct io_uring_params *p)
12
+ int io_uring_enter2(unsigned int fd, unsigned int to_submit,
13
+ unsigned int min_complete, unsigned int flags,
14
+ sigset_t *sig, size_t sz)
31
15
  {
32
- return syscall(__NR_io_uring_setup, entries, p);
16
+ return __sys_io_uring_enter2(fd, to_submit, min_complete, flags, sig,
17
+ sz);
33
18
  }
34
19
 
35
- int __sys_io_uring_enter2(int fd, unsigned to_submit, unsigned min_complete,
36
- unsigned flags, sigset_t *sig, int sz)
20
+ int io_uring_setup(unsigned int entries, struct io_uring_params *p)
37
21
  {
38
- return syscall(__NR_io_uring_enter, fd, to_submit, min_complete, flags,
39
- sig, sz);
22
+ return __sys_io_uring_setup(entries, p);
40
23
  }
41
24
 
42
- int __sys_io_uring_enter(int fd, unsigned to_submit, unsigned min_complete,
43
- unsigned flags, sigset_t *sig)
25
+ int io_uring_register(unsigned int fd, unsigned int opcode, const void *arg,
26
+ unsigned int nr_args)
44
27
  {
45
- return __sys_io_uring_enter2(fd, to_submit, min_complete, flags, sig,
46
- _NSIG / 8);
28
+ return __sys_io_uring_register(fd, opcode, arg, nr_args);
47
29
  }