uringmachine 0.10 → 0.11

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (80) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +4 -0
  3. data/examples/bm_side_running.rb +83 -0
  4. data/examples/bm_sqlite.rb +1 -1
  5. data/ext/um/um.c +17 -1
  6. data/ext/um/um.h +29 -0
  7. data/ext/um/um_ext.c +2 -0
  8. data/ext/um/um_stream.c +344 -0
  9. data/ext/um/um_stream_class.c +140 -0
  10. data/lib/uringmachine/version.rb +1 -1
  11. data/lib/uringmachine.rb +20 -16
  12. data/test/test_stream.rb +133 -0
  13. data/test/test_um.rb +63 -0
  14. data/vendor/liburing/.github/workflows/{build.yml → ci.yml} +107 -42
  15. data/vendor/liburing/.gitignore +1 -0
  16. data/vendor/liburing/CHANGELOG +10 -0
  17. data/vendor/liburing/README +5 -0
  18. data/vendor/liburing/configure +1 -1
  19. data/vendor/liburing/examples/Makefile +1 -0
  20. data/vendor/liburing/examples/helpers.c +25 -0
  21. data/vendor/liburing/examples/helpers.h +13 -0
  22. data/vendor/liburing/examples/io_uring-test.c +3 -0
  23. data/vendor/liburing/examples/proxy.c +1 -1
  24. data/vendor/liburing/examples/reg-wait.c +41 -6
  25. data/vendor/liburing/examples/send-zerocopy.c +79 -32
  26. data/vendor/liburing/examples/zcrx.c +436 -0
  27. data/vendor/liburing/liburing.spec +1 -1
  28. data/vendor/liburing/src/Makefile +0 -1
  29. data/vendor/liburing/src/arch/generic/syscall.h +2 -2
  30. data/vendor/liburing/src/arch/syscall-defs.h +2 -2
  31. data/vendor/liburing/src/include/liburing/io_uring.h +101 -17
  32. data/vendor/liburing/src/include/liburing.h +179 -59
  33. data/vendor/liburing/src/int_flags.h +4 -1
  34. data/vendor/liburing/src/liburing-ffi.map +14 -2
  35. data/vendor/liburing/src/liburing.map +9 -2
  36. data/vendor/liburing/src/queue.c +35 -30
  37. data/vendor/liburing/src/register.c +46 -15
  38. data/vendor/liburing/src/sanitize.c +6 -9
  39. data/vendor/liburing/src/setup.c +37 -71
  40. data/vendor/liburing/src/syscall.c +2 -2
  41. data/vendor/liburing/test/232c93d07b74.c +1 -0
  42. data/vendor/liburing/test/Makefile +9 -0
  43. data/vendor/liburing/test/accept-test.c +1 -0
  44. data/vendor/liburing/test/cmd-discard.c +16 -8
  45. data/vendor/liburing/test/connect.c +11 -7
  46. data/vendor/liburing/test/epwait.c +420 -0
  47. data/vendor/liburing/test/eventfd-ring.c +30 -5
  48. data/vendor/liburing/test/fallocate.c +1 -1
  49. data/vendor/liburing/test/fixed-hugepage.c +10 -7
  50. data/vendor/liburing/test/fixed-seg.c +187 -0
  51. data/vendor/liburing/test/helpers.c +121 -0
  52. data/vendor/liburing/test/helpers.h +13 -0
  53. data/vendor/liburing/test/init-mem.c +2 -0
  54. data/vendor/liburing/test/io_uring_passthrough.c +78 -62
  55. data/vendor/liburing/test/iopoll-overflow.c +5 -4
  56. data/vendor/liburing/test/iopoll.c +20 -10
  57. data/vendor/liburing/test/iowait.c +141 -0
  58. data/vendor/liburing/test/nvme.h +2 -0
  59. data/vendor/liburing/test/pipe-bug.c +11 -5
  60. data/vendor/liburing/test/pipe-eof.c +11 -1
  61. data/vendor/liburing/test/read-inc-file.c +150 -0
  62. data/vendor/liburing/test/read-write.c +21 -14
  63. data/vendor/liburing/test/recv-bundle-short-ooo.c +435 -0
  64. data/vendor/liburing/test/recv-multishot.c +2 -2
  65. data/vendor/liburing/test/reg-wait.c +449 -120
  66. data/vendor/liburing/test/regbuf-clone.c +53 -0
  67. data/vendor/liburing/test/resize-rings.c +25 -2
  68. data/vendor/liburing/test/rsrc_tags.c +67 -14
  69. data/vendor/liburing/test/send-zerocopy.c +52 -130
  70. data/vendor/liburing/test/sendmsg_iov_clean.c +216 -0
  71. data/vendor/liburing/test/socket-nb.c +158 -0
  72. data/vendor/liburing/test/sqwait.c +9 -11
  73. data/vendor/liburing/test/timeout.c +198 -0
  74. data/vendor/liburing/test/vec-regbuf.c +609 -0
  75. data/vendor/liburing/test/wait-timeout.c +1 -1
  76. data/vendor/liburing/test/wq-aff.c +5 -1
  77. data/vendor/liburing/test/zcrx.c +928 -0
  78. metadata +16 -4
  79. data/vendor/liburing/.github/workflows/codespell.yml +0 -25
  80. data/vendor/liburing/.github/workflows/shellcheck.yml +0 -20
@@ -47,7 +47,7 @@ static inline bool cq_ring_needs_flush(struct io_uring *ring)
47
47
 
48
48
  static inline bool cq_ring_needs_enter(struct io_uring *ring)
49
49
  {
50
- return (ring->flags & IORING_SETUP_IOPOLL) || cq_ring_needs_flush(ring);
50
+ return (ring->int_flags & INT_FLAG_CQ_ENTER) || cq_ring_needs_flush(ring);
51
51
  }
52
52
 
53
53
  struct get_data {
@@ -151,6 +151,31 @@ int io_uring_get_events(struct io_uring *ring)
151
151
  return __sys_io_uring_enter(ring->enter_ring_fd, 0, 0, flags, NULL);
152
152
  }
153
153
 
154
+ static inline bool io_uring_peek_batch_cqe_(struct io_uring *ring,
155
+ struct io_uring_cqe **cqes,
156
+ unsigned *count)
157
+ {
158
+ unsigned ready = io_uring_cq_ready(ring);
159
+ unsigned shift;
160
+ unsigned head;
161
+ unsigned mask;
162
+ unsigned last;
163
+
164
+ if (!ready)
165
+ return false;
166
+
167
+ shift = io_uring_cqe_shift(ring);
168
+ head = *ring->cq.khead;
169
+ mask = ring->cq.ring_mask;
170
+ if (ready < *count)
171
+ *count = ready;
172
+ last = head + *count;
173
+ for (;head != last; head++)
174
+ *(cqes++) = &ring->cq.cqes[(head & mask) << shift];
175
+
176
+ return true;
177
+ }
178
+
154
179
  /*
155
180
  * Fill in an array of IO completions up to count, if any are available.
156
181
  * Returns the amount of IO completions filled.
@@ -158,39 +183,17 @@ int io_uring_get_events(struct io_uring *ring)
158
183
  unsigned io_uring_peek_batch_cqe(struct io_uring *ring,
159
184
  struct io_uring_cqe **cqes, unsigned count)
160
185
  {
161
- unsigned ready;
162
- bool overflow_checked = false;
163
- int shift = 0;
164
-
165
- if (ring->flags & IORING_SETUP_CQE32)
166
- shift = 1;
167
-
168
- again:
169
- ready = io_uring_cq_ready(ring);
170
- if (ready) {
171
- unsigned head = *ring->cq.khead;
172
- unsigned mask = ring->cq.ring_mask;
173
- unsigned last;
174
- int i = 0;
175
-
176
- count = count > ready ? ready : count;
177
- last = head + count;
178
- for (;head != last; head++, i++)
179
- cqes[i] = &ring->cq.cqes[(head & mask) << shift];
180
-
186
+ if (io_uring_peek_batch_cqe_(ring, cqes, &count))
181
187
  return count;
182
- }
183
188
 
184
- if (overflow_checked)
189
+ if (!cq_ring_needs_flush(ring))
185
190
  return 0;
186
191
 
187
- if (cq_ring_needs_flush(ring)) {
188
- io_uring_get_events(ring);
189
- overflow_checked = true;
190
- goto again;
191
- }
192
+ io_uring_get_events(ring);
193
+ if (!io_uring_peek_batch_cqe_(ring, cqes, &count))
194
+ return 0;
192
195
 
193
- return 0;
196
+ return count;
194
197
  }
195
198
 
196
199
  /*
@@ -324,6 +327,8 @@ int io_uring_submit_and_wait_reg(struct io_uring *ring,
324
327
  struct io_uring_cqe **cqe_ptr,
325
328
  unsigned wait_nr, int reg_index)
326
329
  {
330
+ unsigned long offset = reg_index * sizeof(struct io_uring_reg_wait);
331
+
327
332
  struct get_data data = {
328
333
  .submit = __io_uring_flush_sq(ring),
329
334
  .wait_nr = wait_nr,
@@ -331,7 +336,7 @@ int io_uring_submit_and_wait_reg(struct io_uring *ring,
331
336
  IORING_ENTER_EXT_ARG_REG,
332
337
  .sz = sizeof(struct io_uring_reg_wait),
333
338
  .has_ts = true,
334
- .arg = (void *) (uintptr_t) reg_index,
339
+ .arg = (void *) (uintptr_t) offset,
335
340
  };
336
341
 
337
342
  if (!(ring->features & IORING_FEAT_EXT_ARG))
@@ -111,9 +111,9 @@ int io_uring_register_files_update(struct io_uring *ring, unsigned off,
111
111
  {
112
112
  liburing_sanitize_address(files);
113
113
 
114
- struct io_uring_files_update up = {
114
+ struct io_uring_rsrc_update up = {
115
115
  .offset = off,
116
- .fds = (unsigned long) files,
116
+ .data = (unsigned long) files,
117
117
  };
118
118
 
119
119
  return do_register(ring, IORING_REGISTER_FILES_UPDATE, &up, nr_files);
@@ -395,9 +395,9 @@ int io_uring_register_clock(struct io_uring *ring,
395
395
  return do_register(ring, IORING_REGISTER_CLOCK, arg, 0);
396
396
  }
397
397
 
398
- int io_uring_clone_buffers_offset(struct io_uring *dst, struct io_uring *src,
399
- unsigned int dst_off, unsigned int src_off,
400
- unsigned int nr, unsigned int flags)
398
+ int __io_uring_clone_buffers_offset(struct io_uring *dst, struct io_uring *src,
399
+ unsigned int dst_off, unsigned int src_off,
400
+ unsigned int nr, unsigned int flags)
401
401
  {
402
402
  struct io_uring_clone_buffers buf = {
403
403
  .src_fd = src->ring_fd,
@@ -407,19 +407,40 @@ int io_uring_clone_buffers_offset(struct io_uring *dst, struct io_uring *src,
407
407
  .nr = nr,
408
408
  };
409
409
 
410
- if (src->int_flags & INT_FLAG_REG_REG_RING) {
410
+ if (flags & IORING_REGISTER_SRC_REGISTERED &&
411
+ src->int_flags & INT_FLAG_REG_REG_RING) {
411
412
  buf.src_fd = src->enter_ring_fd;
412
- buf.flags |= IORING_REGISTER_SRC_REGISTERED;
413
413
  } else {
414
414
  buf.src_fd = src->ring_fd;
415
+ buf.flags &= ~IORING_REGISTER_SRC_REGISTERED;
415
416
  }
416
417
 
417
418
  return do_register(dst, IORING_REGISTER_CLONE_BUFFERS, &buf, 1);
418
419
  }
419
420
 
421
+ int io_uring_clone_buffers_offset(struct io_uring *dst, struct io_uring *src,
422
+ unsigned int dst_off, unsigned int src_off,
423
+ unsigned int nr, unsigned int flags)
424
+ {
425
+ return __io_uring_clone_buffers_offset(dst, src, dst_off, src_off, nr,
426
+ flags | IORING_REGISTER_SRC_REGISTERED);
427
+ }
428
+
420
429
  int io_uring_clone_buffers(struct io_uring *dst, struct io_uring *src)
421
430
  {
422
- return io_uring_clone_buffers_offset(dst, src, 0, 0, 0, 0);
431
+ return __io_uring_clone_buffers_offset(dst, src, 0, 0, 0, IORING_REGISTER_SRC_REGISTERED);
432
+ }
433
+
434
+ int __io_uring_clone_buffers(struct io_uring *dst, struct io_uring *src,
435
+ unsigned int flags)
436
+ {
437
+ return __io_uring_clone_buffers_offset(dst, src, 0, 0, 0, flags);
438
+ }
439
+
440
+ int io_uring_register_ifq(struct io_uring *ring,
441
+ struct io_uring_zcrx_ifq_reg *reg)
442
+ {
443
+ return do_register(ring, IORING_REGISTER_ZCRX_IFQ, reg, 1);
423
444
  }
424
445
 
425
446
  int io_uring_resize_rings(struct io_uring *ring, struct io_uring_params *p)
@@ -468,12 +489,22 @@ out:
468
489
  int io_uring_register_wait_reg(struct io_uring *ring,
469
490
  struct io_uring_reg_wait *reg, int nr)
470
491
  {
471
- struct io_uring_cqwait_reg_arg arg = {
472
- .flags = 0,
473
- .struct_size = sizeof(*reg),
474
- .nr_entries = nr,
475
- .user_addr = (unsigned long) (uintptr_t) reg,
476
- };
492
+ return -EINVAL;
493
+ }
477
494
 
478
- return do_register(ring, IORING_REGISTER_CQWAIT_REG, &arg, 1);
495
+ int io_uring_register_region(struct io_uring *ring,
496
+ struct io_uring_mem_region_reg *reg)
497
+ {
498
+ return do_register(ring, IORING_REGISTER_MEM_REGION, reg, 1);
499
+ }
500
+
501
+ int io_uring_set_iowait(struct io_uring *ring, bool enable_iowait)
502
+ {
503
+ if (!(ring->features & IORING_FEAT_NO_IOWAIT))
504
+ return -EOPNOTSUPP;
505
+ if (enable_iowait)
506
+ ring->int_flags &= ~INT_FLAG_NO_IOWAIT;
507
+ else
508
+ ring->int_flags |= INT_FLAG_NO_IOWAIT;
509
+ return 0;
479
510
  }
@@ -115,6 +115,10 @@ static inline void initialize_sanitize_handlers()
115
115
  sanitize_handlers[IORING_OP_FTRUNCATE] = sanitize_sqe_addr;
116
116
  sanitize_handlers[IORING_OP_BIND] = sanitize_sqe_addr;
117
117
  sanitize_handlers[IORING_OP_LISTEN] = sanitize_sqe_addr;
118
+ sanitize_handlers[IORING_OP_RECV_ZC] = sanitize_sqe_addr,
119
+ sanitize_handlers[IORING_OP_EPOLL_WAIT] = sanitize_sqe_addr,
120
+ sanitize_handlers[IORING_OP_READV_FIXED] = sanitize_sqe_addr,
121
+ sanitize_handlers[IORING_OP_WRITEV_FIXED] = sanitize_sqe_addr,
118
122
  sanitize_handlers_initialized = true;
119
123
  }
120
124
 
@@ -122,18 +126,11 @@ void liburing_sanitize_ring(struct io_uring *ring)
122
126
  {
123
127
  struct io_uring_sq *sq = &ring->sq;
124
128
  struct io_uring_sqe *sqe;
125
- unsigned int head;
126
- int shift = 0;
129
+ unsigned head = io_uring_load_sq_head(ring);
130
+ unsigned shift = io_uring_sqe_shift(ring);
127
131
 
128
132
  initialize_sanitize_handlers();
129
133
 
130
- if (ring->flags & IORING_SETUP_SQE128)
131
- shift = 1;
132
- if (!(ring->flags & IORING_SETUP_SQPOLL))
133
- head = *sq->khead;
134
- else
135
- head = io_uring_smp_load_acquire(sq->khead);
136
-
137
134
  while (head != sq->sqe_tail) {
138
135
  sqe = &sq->sqes[(head & sq->ring_mask) << shift];
139
136
  if (sqe->opcode < IORING_OP_LAST)
@@ -94,18 +94,25 @@ void io_uring_setup_ring_pointers(struct io_uring_params *p,
94
94
  cq->ring_entries = *cq->kring_entries;
95
95
  }
96
96
 
97
+ static size_t params_sqes_size(const struct io_uring_params *p, unsigned sqes)
98
+ {
99
+ sqes <<= io_uring_sqe_shift_from_flags(p->flags);
100
+ return sqes * sizeof(struct io_uring_sqe);
101
+ }
102
+
103
+ static size_t params_cq_size(const struct io_uring_params *p, unsigned cqes)
104
+ {
105
+ cqes <<= io_uring_cqe_shift_from_flags(p->flags);
106
+ return cqes * sizeof(struct io_uring_cqe);
107
+ }
108
+
97
109
  int io_uring_mmap(int fd, struct io_uring_params *p, struct io_uring_sq *sq,
98
110
  struct io_uring_cq *cq)
99
111
  {
100
- size_t size;
101
112
  int ret;
102
113
 
103
- size = sizeof(struct io_uring_cqe);
104
- if (p->flags & IORING_SETUP_CQE32)
105
- size += sizeof(struct io_uring_cqe);
106
-
107
114
  sq->ring_sz = p->sq_off.array + p->sq_entries * sizeof(unsigned);
108
- cq->ring_sz = p->cq_off.cqes + p->cq_entries * size;
115
+ cq->ring_sz = p->cq_off.cqes + params_cq_size(p, p->cq_entries);
109
116
 
110
117
  if (p->features & IORING_FEAT_SINGLE_MMAP) {
111
118
  if (cq->ring_sz > sq->ring_sz)
@@ -131,11 +138,9 @@ int io_uring_mmap(int fd, struct io_uring_params *p, struct io_uring_sq *sq,
131
138
  }
132
139
  }
133
140
 
134
- size = sizeof(struct io_uring_sqe);
135
- if (p->flags & IORING_SETUP_SQE128)
136
- size += 64;
137
- sq->sqes = __sys_mmap(0, size * p->sq_entries, PROT_READ | PROT_WRITE,
138
- MAP_SHARED | MAP_POPULATE, fd, IORING_OFF_SQES);
141
+ sq->sqes = __sys_mmap(0, params_sqes_size(p, p->sq_entries),
142
+ PROT_READ | PROT_WRITE, MAP_SHARED | MAP_POPULATE,
143
+ fd, IORING_OFF_SQES);
139
144
  if (IS_ERR(sq->sqes)) {
140
145
  ret = PTR_ERR(sq->sqes);
141
146
  err:
@@ -160,6 +165,12 @@ __cold int io_uring_queue_mmap(int fd, struct io_uring_params *p,
160
165
  return io_uring_mmap(fd, p, &ring->sq, &ring->cq);
161
166
  }
162
167
 
168
+ static size_t io_uring_sqes_size(const struct io_uring *ring)
169
+ {
170
+ return (ring->sq.ring_entries << io_uring_sqe_shift(ring)) *
171
+ sizeof(struct io_uring_sqe);
172
+ }
173
+
163
174
  /*
164
175
  * Ensure that the mmap'ed rings aren't available to a child after a fork(2).
165
176
  * This uses madvise(..., MADV_DONTFORK) on the mmap'ed ranges.
@@ -172,10 +183,7 @@ __cold int io_uring_ring_dontfork(struct io_uring *ring)
172
183
  if (!ring->sq.ring_ptr || !ring->sq.sqes || !ring->cq.ring_ptr)
173
184
  return -EINVAL;
174
185
 
175
- len = sizeof(struct io_uring_sqe);
176
- if (ring->flags & IORING_SETUP_SQE128)
177
- len += 64;
178
- len *= ring->sq.ring_entries;
186
+ len = io_uring_sqes_size(ring);
179
187
  ret = __sys_madvise(ring->sq.sqes, len, MADV_DONTFORK);
180
188
  if (ret < 0)
181
189
  return ret;
@@ -209,7 +217,7 @@ static int io_uring_alloc_huge(unsigned entries, struct io_uring_params *p,
209
217
  {
210
218
  unsigned long page_size = get_page_size();
211
219
  unsigned sq_entries, cq_entries;
212
- size_t ring_mem, sqes_mem, cqes_mem;
220
+ size_t ring_mem, sqes_mem;
213
221
  unsigned long mem_used = 0;
214
222
  void *ptr;
215
223
  int ret;
@@ -220,15 +228,12 @@ static int io_uring_alloc_huge(unsigned entries, struct io_uring_params *p,
220
228
 
221
229
  ring_mem = KRING_SIZE;
222
230
 
223
- sqes_mem = sq_entries * sizeof(struct io_uring_sqe);
231
+ sqes_mem = params_sqes_size(p, sq_entries);
224
232
  if (!(p->flags & IORING_SETUP_NO_SQARRAY))
225
233
  sqes_mem += sq_entries * sizeof(unsigned);
226
234
  sqes_mem = (sqes_mem + page_size - 1) & ~(page_size - 1);
227
235
 
228
- cqes_mem = cq_entries * sizeof(struct io_uring_cqe);
229
- if (p->flags & IORING_SETUP_CQE32)
230
- cqes_mem *= 2;
231
- ring_mem += sqes_mem + cqes_mem;
236
+ ring_mem += sqes_mem + params_cq_size(p, cq_entries);
232
237
  mem_used = ring_mem;
233
238
  mem_used = (mem_used + page_size - 1) & ~(page_size - 1);
234
239
 
@@ -359,6 +364,13 @@ int __io_uring_queue_init_params(unsigned entries, struct io_uring *ring,
359
364
  } else {
360
365
  ring->ring_fd = fd;
361
366
  }
367
+ /*
368
+ * IOPOLL always needs to enter, except if SQPOLL is set as well.
369
+ * Use an internal flag to check for this.
370
+ */
371
+ if ((ring->flags & (IORING_SETUP_IOPOLL|IORING_SETUP_SQPOLL)) ==
372
+ IORING_SETUP_IOPOLL)
373
+ ring->int_flags |= INT_FLAG_CQ_ENTER;
362
374
 
363
375
  return ret;
364
376
  }
@@ -430,20 +442,10 @@ __cold void io_uring_queue_exit(struct io_uring *ring)
430
442
  {
431
443
  struct io_uring_sq *sq = &ring->sq;
432
444
  struct io_uring_cq *cq = &ring->cq;
433
- size_t sqe_size;
434
445
 
435
- if (!sq->ring_sz && !(ring->int_flags & INT_FLAG_APP_MEM)) {
436
- sqe_size = sizeof(struct io_uring_sqe);
437
- if (ring->flags & IORING_SETUP_SQE128)
438
- sqe_size += 64;
439
- __sys_munmap(sq->sqes, sqe_size * sq->ring_entries);
446
+ if (!(ring->int_flags & INT_FLAG_APP_MEM)) {
447
+ __sys_munmap(sq->sqes, io_uring_sqes_size(ring));
440
448
  io_uring_unmap_rings(sq, cq);
441
- } else {
442
- if (!(ring->int_flags & INT_FLAG_APP_MEM)) {
443
- __sys_munmap(sq->sqes,
444
- *sq->kring_entries * sizeof(struct io_uring_sqe));
445
- io_uring_unmap_rings(sq, cq);
446
- }
447
449
  }
448
450
 
449
451
  /*
@@ -508,18 +510,11 @@ static size_t rings_size(struct io_uring_params *p, unsigned entries,
508
510
  {
509
511
  size_t pages, sq_size, cq_size;
510
512
 
511
- cq_size = sizeof(struct io_uring_cqe);
512
- if (p->flags & IORING_SETUP_CQE32)
513
- cq_size += sizeof(struct io_uring_cqe);
514
- cq_size *= cq_entries;
515
- cq_size += KRING_SIZE;
513
+ cq_size = KRING_SIZE + params_cq_size(p, cq_entries);
516
514
  cq_size = (cq_size + 63) & ~63UL;
517
515
  pages = (size_t) 1 << npages(cq_size, page_size);
518
516
 
519
- sq_size = sizeof(struct io_uring_sqe);
520
- if (p->flags & IORING_SETUP_SQE128)
521
- sq_size += 64;
522
- sq_size *= entries;
517
+ sq_size = params_sqes_size(p, entries);
523
518
  pages += (size_t) 1 << npages(sq_size, page_size);
524
519
  return pages * page_size;
525
520
  }
@@ -686,32 +681,3 @@ int io_uring_free_buf_ring(struct io_uring *ring, struct io_uring_buf_ring *br,
686
681
  __sys_munmap(br, nentries * sizeof(struct io_uring_buf));
687
682
  return 0;
688
683
  }
689
-
690
- void io_uring_free_reg_wait(struct io_uring_reg_wait *reg, unsigned nentries)
691
- {
692
- __sys_munmap(reg, nentries * sizeof(struct io_uring_reg_wait));
693
- }
694
-
695
- struct io_uring_reg_wait *io_uring_setup_reg_wait(struct io_uring *ring,
696
- unsigned nentries, int *err)
697
- {
698
- struct io_uring_reg_wait *reg;
699
- size_t size = nentries * sizeof(*reg);
700
- int ret;
701
-
702
- reg = __sys_mmap(NULL, size, PROT_READ | PROT_WRITE,
703
- MAP_SHARED|MAP_POPULATE|MAP_ANONYMOUS, -1, 0);
704
- if (IS_ERR(reg)) {
705
- *err = PTR_ERR(reg);
706
- return NULL;
707
- }
708
-
709
- memset(reg, 0, size);
710
- ret = io_uring_register_wait_reg(ring, reg, nentries);
711
- if (!ret)
712
- return reg;
713
-
714
- __sys_munmap(reg, size);
715
- *err = ret;
716
- return NULL;
717
- }
@@ -11,9 +11,9 @@ int io_uring_enter(unsigned int fd, unsigned int to_submit,
11
11
 
12
12
  int io_uring_enter2(unsigned int fd, unsigned int to_submit,
13
13
  unsigned int min_complete, unsigned int flags,
14
- sigset_t *sig, size_t sz)
14
+ void *arg, size_t sz)
15
15
  {
16
- return __sys_io_uring_enter2(fd, to_submit, min_complete, flags, sig,
16
+ return __sys_io_uring_enter2(fd, to_submit, min_complete, flags, arg,
17
17
  sz);
18
18
  }
19
19
 
@@ -10,6 +10,7 @@
10
10
  #include <stdlib.h>
11
11
  #include <stdint.h>
12
12
  #include <assert.h>
13
+ #include <string.h>
13
14
 
14
15
  #include <pthread.h>
15
16
  #include <errno.h>
@@ -81,6 +81,7 @@ test_srcs := \
81
81
  eeed8b54e0df.c \
82
82
  empty-eownerdead.c \
83
83
  eploop.c \
84
+ epwait.c \
84
85
  eventfd.c \
85
86
  eventfd-disable.c \
86
87
  eventfd-reg.c \
@@ -106,6 +107,7 @@ test_srcs := \
106
107
  fixed-hugepage.c \
107
108
  fixed-link.c \
108
109
  fixed-reuse.c \
110
+ fixed-seg.c \
109
111
  fpos.c \
110
112
  fsnotify.c \
111
113
  fsync.c \
@@ -121,6 +123,7 @@ test_srcs := \
121
123
  io_uring_passthrough.c \
122
124
  io_uring_register.c \
123
125
  io_uring_setup.c \
126
+ iowait.c \
124
127
  kallsyms.c \
125
128
  lfs-openat.c \
126
129
  lfs-openat-write.c \
@@ -165,10 +168,12 @@ test_srcs := \
165
168
  pollfree.c \
166
169
  probe.c \
167
170
  read-before-exit.c \
171
+ read-inc-file.c \
168
172
  read-mshot.c \
169
173
  read-mshot-empty.c \
170
174
  read-mshot-stdin.c \
171
175
  read-write.c \
176
+ recv-bundle-short-ooo.c \
172
177
  recv-msgall.c \
173
178
  recv-msgall-stream.c \
174
179
  recv-multishot.c \
@@ -193,6 +198,7 @@ test_srcs := \
193
198
  send_recv.c \
194
199
  send_recvmsg.c \
195
200
  send-zerocopy.c \
201
+ sendmsg_iov_clean.c \
196
202
  shared-wq.c \
197
203
  short-read.c \
198
204
  shutdown.c \
@@ -202,6 +208,7 @@ test_srcs := \
202
208
  socket.c \
203
209
  socket-io-cmd.c \
204
210
  socket-getsetsock-cmd.c \
211
+ socket-nb.c \
205
212
  socket-rw.c \
206
213
  socket-rw-eagain.c \
207
214
  socket-rw-offset.c \
@@ -237,6 +244,8 @@ test_srcs := \
237
244
  wakeup-hang.c \
238
245
  wq-aff.c \
239
246
  xattr.c \
247
+ zcrx.c \
248
+ vec-regbuf.c \
240
249
  # EOL
241
250
 
242
251
  # Please keep this list sorted alphabetically.
@@ -7,6 +7,7 @@
7
7
  #include <sys/socket.h>
8
8
  #include <sys/un.h>
9
9
  #include <assert.h>
10
+ #include <string.h>
10
11
  #include "liburing.h"
11
12
  #include "helpers.h"
12
13
 
@@ -199,7 +199,7 @@ static int basic_cmd_test(struct io_uring *ring, int op)
199
199
 
200
200
  fd = open(filename, O_DIRECT | O_RDWR | O_EXCL);
201
201
  if (fd < 0) {
202
- if (errno == -EINVAL || errno == -EBUSY)
202
+ if (errno == EINVAL || errno == EBUSY)
203
203
  return T_EXIT_SKIP;
204
204
  fprintf(stderr, "open failed %i\n", errno);
205
205
  return T_EXIT_FAIL;
@@ -248,6 +248,8 @@ static int test_fail_edge_cases(struct io_uring *ring, int op)
248
248
 
249
249
  fd = open(filename, O_DIRECT | O_RDWR | O_EXCL);
250
250
  if (fd < 0) {
251
+ if (errno == EINVAL || errno == EBUSY)
252
+ return T_EXIT_SKIP;
251
253
  fprintf(stderr, "open failed %i\n", errno);
252
254
  return T_EXIT_FAIL;
253
255
  }
@@ -302,6 +304,8 @@ static int test_rdonly(struct io_uring *ring, int op)
302
304
 
303
305
  fd = open(filename, O_DIRECT | O_RDONLY | O_EXCL);
304
306
  if (fd < 0) {
307
+ if (errno == EINVAL || errno == EBUSY)
308
+ return T_EXIT_SKIP;
305
309
  fprintf(stderr, "open failed %i\n", errno);
306
310
  return T_EXIT_FAIL;
307
311
  }
@@ -315,6 +319,8 @@ static int test_rdonly(struct io_uring *ring, int op)
315
319
 
316
320
  fd = open(filename, O_DIRECT | O_RDWR | O_EXCL);
317
321
  if (fd < 0) {
322
+ if (errno == EINVAL || errno == EBUSY)
323
+ return T_EXIT_SKIP;
318
324
  fprintf(stderr, "open failed %i\n", errno);
319
325
  return T_EXIT_FAIL;
320
326
  }
@@ -354,6 +360,8 @@ int main(int argc, char *argv[])
354
360
 
355
361
  fd = open(filename, O_DIRECT | O_RDONLY | O_EXCL);
356
362
  if (fd < 0) {
363
+ if (errno == EINVAL || errno == EBUSY)
364
+ return T_EXIT_SKIP;
357
365
  fprintf(stderr, "open failed %i\n", errno);
358
366
  return T_EXIT_FAIL;
359
367
  }
@@ -396,29 +404,29 @@ int main(int argc, char *argv[])
396
404
  if (!opcodes[cmd_op].test)
397
405
  continue;
398
406
  ret = basic_cmd_test(&ring, cmd_op);
399
- if (ret) {
400
- if (ret == T_EXIT_SKIP)
401
- continue;
402
-
407
+ if (ret == T_EXIT_FAIL) {
403
408
  fprintf(stderr, "basic_cmd_test() failed, cmd %i\n",
404
409
  cmd_op);
405
410
  return T_EXIT_FAIL;
406
411
  }
407
412
 
408
413
  ret = test_rdonly(&ring, cmd_op);
409
- if (ret) {
414
+ if (ret == T_EXIT_FAIL) {
410
415
  fprintf(stderr, "test_rdonly() failed, cmd %i\n",
411
416
  cmd_op);
412
417
  return T_EXIT_FAIL;
413
418
  }
414
419
 
415
420
  ret = test_fail_edge_cases(&ring, cmd_op);
416
- if (ret) {
421
+ if (ret == T_EXIT_FAIL) {
417
422
  fprintf(stderr, "test_fail_edge_cases() failed, cmd %i\n",
418
423
  cmd_op);
419
424
  return T_EXIT_FAIL;
420
425
  }
421
- fret = T_EXIT_PASS;
426
+ if (ret == T_EXIT_SKIP)
427
+ fret = T_EXIT_SKIP;
428
+ else
429
+ fret = T_EXIT_PASS;
422
430
  }
423
431
 
424
432
  io_uring_queue_exit(&ring);
@@ -371,8 +371,10 @@ static int test(int flags)
371
371
  int ret;
372
372
 
373
373
  ret = io_uring_queue_init(8, &ring, flags);
374
+ if (ret == -EINVAL)
375
+ return T_EXIT_SKIP;
374
376
  if (ret) {
375
- fprintf(stderr, "io_uring_queue_setup() = %d\n", ret);
377
+ fprintf(stderr, "io_uring_queue_init() = %d\n", ret);
376
378
  return T_EXIT_FAIL;
377
379
  }
378
380
 
@@ -413,30 +415,32 @@ static int test(int flags)
413
415
 
414
416
  int main(int argc, char *argv[])
415
417
  {
418
+ bool any_passed = false;
416
419
  int ret;
417
420
 
418
421
  if (argc > 1)
419
422
  return T_EXIT_SKIP;
420
423
 
421
424
  ret = test(0);
422
- if (ret == -1) {
425
+ any_passed |= ret == T_EXIT_PASS;
426
+ if (ret == T_EXIT_FAIL) {
423
427
  fprintf(stderr, "test 0 failed\n");
424
428
  return T_EXIT_FAIL;
425
429
  }
426
- if (no_connect)
427
- return T_EXIT_SKIP;
428
430
 
429
431
  ret = test(IORING_SETUP_SQPOLL);
430
- if (ret == -1) {
432
+ any_passed |= ret == T_EXIT_PASS;
433
+ if (ret == T_EXIT_FAIL) {
431
434
  fprintf(stderr, "test SQPOLL failed\n");
432
435
  return T_EXIT_FAIL;
433
436
  }
434
437
 
435
438
  ret = test(IORING_SETUP_SINGLE_ISSUER|IORING_SETUP_DEFER_TASKRUN);
436
- if (ret == -1) {
439
+ any_passed |= ret == T_EXIT_PASS;
440
+ if (ret == T_EXIT_FAIL) {
437
441
  fprintf(stderr, "test DEFER failed\n");
438
442
  return T_EXIT_FAIL;
439
443
  }
440
444
 
441
- return T_EXIT_PASS;
445
+ return any_passed ? T_EXIT_PASS : T_EXIT_SKIP;
442
446
  }