uringmachine 0.8.2 → 0.11
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +13 -0
- data/TODO.md +0 -1
- data/examples/bm_side_running.rb +83 -0
- data/examples/bm_sqlite.rb +1 -1
- data/ext/um/um.c +66 -4
- data/ext/um/um.h +36 -0
- data/ext/um/um_class.c +6 -0
- data/ext/um/um_const.c +36 -0
- data/ext/um/um_ext.c +2 -0
- data/ext/um/um_stream.c +344 -0
- data/ext/um/um_stream_class.c +140 -0
- data/ext/um/um_utils.c +4 -0
- data/lib/uringmachine/actor.rb +1 -1
- data/lib/uringmachine/version.rb +1 -1
- data/lib/uringmachine.rb +35 -17
- data/test/test_fiber.rb +23 -3
- data/test/test_stream.rb +133 -0
- data/test/test_um.rb +109 -2
- data/uringmachine.gemspec +0 -2
- data/vendor/liburing/.github/workflows/{build.yml → ci.yml} +107 -42
- data/vendor/liburing/.gitignore +1 -0
- data/vendor/liburing/CHANGELOG +10 -0
- data/vendor/liburing/README +5 -0
- data/vendor/liburing/configure +1 -1
- data/vendor/liburing/examples/Makefile +1 -0
- data/vendor/liburing/examples/helpers.c +25 -0
- data/vendor/liburing/examples/helpers.h +13 -0
- data/vendor/liburing/examples/io_uring-test.c +3 -0
- data/vendor/liburing/examples/proxy.c +1 -1
- data/vendor/liburing/examples/reg-wait.c +41 -6
- data/vendor/liburing/examples/send-zerocopy.c +79 -32
- data/vendor/liburing/examples/zcrx.c +436 -0
- data/vendor/liburing/liburing.spec +1 -1
- data/vendor/liburing/src/Makefile +0 -1
- data/vendor/liburing/src/arch/generic/syscall.h +2 -2
- data/vendor/liburing/src/arch/syscall-defs.h +2 -2
- data/vendor/liburing/src/include/liburing/io_uring.h +101 -17
- data/vendor/liburing/src/include/liburing.h +179 -59
- data/vendor/liburing/src/int_flags.h +4 -1
- data/vendor/liburing/src/liburing-ffi.map +14 -2
- data/vendor/liburing/src/liburing.map +9 -2
- data/vendor/liburing/src/queue.c +35 -30
- data/vendor/liburing/src/register.c +46 -15
- data/vendor/liburing/src/sanitize.c +6 -9
- data/vendor/liburing/src/setup.c +37 -71
- data/vendor/liburing/src/syscall.c +2 -2
- data/vendor/liburing/test/232c93d07b74.c +1 -0
- data/vendor/liburing/test/Makefile +9 -0
- data/vendor/liburing/test/accept-test.c +1 -0
- data/vendor/liburing/test/cmd-discard.c +16 -8
- data/vendor/liburing/test/connect.c +11 -7
- data/vendor/liburing/test/epwait.c +420 -0
- data/vendor/liburing/test/eventfd-ring.c +30 -5
- data/vendor/liburing/test/fallocate.c +1 -1
- data/vendor/liburing/test/fixed-hugepage.c +10 -7
- data/vendor/liburing/test/fixed-seg.c +187 -0
- data/vendor/liburing/test/helpers.c +121 -0
- data/vendor/liburing/test/helpers.h +13 -0
- data/vendor/liburing/test/init-mem.c +2 -0
- data/vendor/liburing/test/io_uring_passthrough.c +78 -62
- data/vendor/liburing/test/iopoll-overflow.c +5 -4
- data/vendor/liburing/test/iopoll.c +20 -10
- data/vendor/liburing/test/iowait.c +141 -0
- data/vendor/liburing/test/nvme.h +2 -0
- data/vendor/liburing/test/pipe-bug.c +11 -5
- data/vendor/liburing/test/pipe-eof.c +11 -1
- data/vendor/liburing/test/read-inc-file.c +150 -0
- data/vendor/liburing/test/read-write.c +21 -14
- data/vendor/liburing/test/recv-bundle-short-ooo.c +435 -0
- data/vendor/liburing/test/recv-multishot.c +2 -2
- data/vendor/liburing/test/reg-wait.c +449 -120
- data/vendor/liburing/test/regbuf-clone.c +53 -0
- data/vendor/liburing/test/resize-rings.c +25 -2
- data/vendor/liburing/test/rsrc_tags.c +67 -14
- data/vendor/liburing/test/send-zerocopy.c +52 -130
- data/vendor/liburing/test/sendmsg_iov_clean.c +216 -0
- data/vendor/liburing/test/socket-nb.c +158 -0
- data/vendor/liburing/test/sqwait.c +9 -11
- data/vendor/liburing/test/timeout.c +198 -0
- data/vendor/liburing/test/vec-regbuf.c +609 -0
- data/vendor/liburing/test/wait-timeout.c +1 -1
- data/vendor/liburing/test/wq-aff.c +5 -1
- data/vendor/liburing/test/zcrx.c +928 -0
- metadata +16 -32
- data/vendor/liburing/.github/workflows/codespell.yml +0 -25
- data/vendor/liburing/.github/workflows/shellcheck.yml +0 -20
data/vendor/liburing/src/queue.c
CHANGED
@@ -47,7 +47,7 @@ static inline bool cq_ring_needs_flush(struct io_uring *ring)
|
|
47
47
|
|
48
48
|
static inline bool cq_ring_needs_enter(struct io_uring *ring)
|
49
49
|
{
|
50
|
-
return (ring->
|
50
|
+
return (ring->int_flags & INT_FLAG_CQ_ENTER) || cq_ring_needs_flush(ring);
|
51
51
|
}
|
52
52
|
|
53
53
|
struct get_data {
|
@@ -151,6 +151,31 @@ int io_uring_get_events(struct io_uring *ring)
|
|
151
151
|
return __sys_io_uring_enter(ring->enter_ring_fd, 0, 0, flags, NULL);
|
152
152
|
}
|
153
153
|
|
154
|
+
static inline bool io_uring_peek_batch_cqe_(struct io_uring *ring,
|
155
|
+
struct io_uring_cqe **cqes,
|
156
|
+
unsigned *count)
|
157
|
+
{
|
158
|
+
unsigned ready = io_uring_cq_ready(ring);
|
159
|
+
unsigned shift;
|
160
|
+
unsigned head;
|
161
|
+
unsigned mask;
|
162
|
+
unsigned last;
|
163
|
+
|
164
|
+
if (!ready)
|
165
|
+
return false;
|
166
|
+
|
167
|
+
shift = io_uring_cqe_shift(ring);
|
168
|
+
head = *ring->cq.khead;
|
169
|
+
mask = ring->cq.ring_mask;
|
170
|
+
if (ready < *count)
|
171
|
+
*count = ready;
|
172
|
+
last = head + *count;
|
173
|
+
for (;head != last; head++)
|
174
|
+
*(cqes++) = &ring->cq.cqes[(head & mask) << shift];
|
175
|
+
|
176
|
+
return true;
|
177
|
+
}
|
178
|
+
|
154
179
|
/*
|
155
180
|
* Fill in an array of IO completions up to count, if any are available.
|
156
181
|
* Returns the amount of IO completions filled.
|
@@ -158,39 +183,17 @@ int io_uring_get_events(struct io_uring *ring)
|
|
158
183
|
unsigned io_uring_peek_batch_cqe(struct io_uring *ring,
|
159
184
|
struct io_uring_cqe **cqes, unsigned count)
|
160
185
|
{
|
161
|
-
|
162
|
-
bool overflow_checked = false;
|
163
|
-
int shift = 0;
|
164
|
-
|
165
|
-
if (ring->flags & IORING_SETUP_CQE32)
|
166
|
-
shift = 1;
|
167
|
-
|
168
|
-
again:
|
169
|
-
ready = io_uring_cq_ready(ring);
|
170
|
-
if (ready) {
|
171
|
-
unsigned head = *ring->cq.khead;
|
172
|
-
unsigned mask = ring->cq.ring_mask;
|
173
|
-
unsigned last;
|
174
|
-
int i = 0;
|
175
|
-
|
176
|
-
count = count > ready ? ready : count;
|
177
|
-
last = head + count;
|
178
|
-
for (;head != last; head++, i++)
|
179
|
-
cqes[i] = &ring->cq.cqes[(head & mask) << shift];
|
180
|
-
|
186
|
+
if (io_uring_peek_batch_cqe_(ring, cqes, &count))
|
181
187
|
return count;
|
182
|
-
}
|
183
188
|
|
184
|
-
if (
|
189
|
+
if (!cq_ring_needs_flush(ring))
|
185
190
|
return 0;
|
186
191
|
|
187
|
-
|
188
|
-
|
189
|
-
|
190
|
-
goto again;
|
191
|
-
}
|
192
|
+
io_uring_get_events(ring);
|
193
|
+
if (!io_uring_peek_batch_cqe_(ring, cqes, &count))
|
194
|
+
return 0;
|
192
195
|
|
193
|
-
return
|
196
|
+
return count;
|
194
197
|
}
|
195
198
|
|
196
199
|
/*
|
@@ -324,6 +327,8 @@ int io_uring_submit_and_wait_reg(struct io_uring *ring,
|
|
324
327
|
struct io_uring_cqe **cqe_ptr,
|
325
328
|
unsigned wait_nr, int reg_index)
|
326
329
|
{
|
330
|
+
unsigned long offset = reg_index * sizeof(struct io_uring_reg_wait);
|
331
|
+
|
327
332
|
struct get_data data = {
|
328
333
|
.submit = __io_uring_flush_sq(ring),
|
329
334
|
.wait_nr = wait_nr,
|
@@ -331,7 +336,7 @@ int io_uring_submit_and_wait_reg(struct io_uring *ring,
|
|
331
336
|
IORING_ENTER_EXT_ARG_REG,
|
332
337
|
.sz = sizeof(struct io_uring_reg_wait),
|
333
338
|
.has_ts = true,
|
334
|
-
.arg = (void *) (uintptr_t)
|
339
|
+
.arg = (void *) (uintptr_t) offset,
|
335
340
|
};
|
336
341
|
|
337
342
|
if (!(ring->features & IORING_FEAT_EXT_ARG))
|
@@ -111,9 +111,9 @@ int io_uring_register_files_update(struct io_uring *ring, unsigned off,
|
|
111
111
|
{
|
112
112
|
liburing_sanitize_address(files);
|
113
113
|
|
114
|
-
struct
|
114
|
+
struct io_uring_rsrc_update up = {
|
115
115
|
.offset = off,
|
116
|
-
.
|
116
|
+
.data = (unsigned long) files,
|
117
117
|
};
|
118
118
|
|
119
119
|
return do_register(ring, IORING_REGISTER_FILES_UPDATE, &up, nr_files);
|
@@ -395,9 +395,9 @@ int io_uring_register_clock(struct io_uring *ring,
|
|
395
395
|
return do_register(ring, IORING_REGISTER_CLOCK, arg, 0);
|
396
396
|
}
|
397
397
|
|
398
|
-
int
|
399
|
-
|
400
|
-
|
398
|
+
int __io_uring_clone_buffers_offset(struct io_uring *dst, struct io_uring *src,
|
399
|
+
unsigned int dst_off, unsigned int src_off,
|
400
|
+
unsigned int nr, unsigned int flags)
|
401
401
|
{
|
402
402
|
struct io_uring_clone_buffers buf = {
|
403
403
|
.src_fd = src->ring_fd,
|
@@ -407,19 +407,40 @@ int io_uring_clone_buffers_offset(struct io_uring *dst, struct io_uring *src,
|
|
407
407
|
.nr = nr,
|
408
408
|
};
|
409
409
|
|
410
|
-
if (
|
410
|
+
if (flags & IORING_REGISTER_SRC_REGISTERED &&
|
411
|
+
src->int_flags & INT_FLAG_REG_REG_RING) {
|
411
412
|
buf.src_fd = src->enter_ring_fd;
|
412
|
-
buf.flags |= IORING_REGISTER_SRC_REGISTERED;
|
413
413
|
} else {
|
414
414
|
buf.src_fd = src->ring_fd;
|
415
|
+
buf.flags &= ~IORING_REGISTER_SRC_REGISTERED;
|
415
416
|
}
|
416
417
|
|
417
418
|
return do_register(dst, IORING_REGISTER_CLONE_BUFFERS, &buf, 1);
|
418
419
|
}
|
419
420
|
|
421
|
+
int io_uring_clone_buffers_offset(struct io_uring *dst, struct io_uring *src,
|
422
|
+
unsigned int dst_off, unsigned int src_off,
|
423
|
+
unsigned int nr, unsigned int flags)
|
424
|
+
{
|
425
|
+
return __io_uring_clone_buffers_offset(dst, src, dst_off, src_off, nr,
|
426
|
+
flags | IORING_REGISTER_SRC_REGISTERED);
|
427
|
+
}
|
428
|
+
|
420
429
|
int io_uring_clone_buffers(struct io_uring *dst, struct io_uring *src)
|
421
430
|
{
|
422
|
-
return
|
431
|
+
return __io_uring_clone_buffers_offset(dst, src, 0, 0, 0, IORING_REGISTER_SRC_REGISTERED);
|
432
|
+
}
|
433
|
+
|
434
|
+
int __io_uring_clone_buffers(struct io_uring *dst, struct io_uring *src,
|
435
|
+
unsigned int flags)
|
436
|
+
{
|
437
|
+
return __io_uring_clone_buffers_offset(dst, src, 0, 0, 0, flags);
|
438
|
+
}
|
439
|
+
|
440
|
+
int io_uring_register_ifq(struct io_uring *ring,
|
441
|
+
struct io_uring_zcrx_ifq_reg *reg)
|
442
|
+
{
|
443
|
+
return do_register(ring, IORING_REGISTER_ZCRX_IFQ, reg, 1);
|
423
444
|
}
|
424
445
|
|
425
446
|
int io_uring_resize_rings(struct io_uring *ring, struct io_uring_params *p)
|
@@ -468,12 +489,22 @@ out:
|
|
468
489
|
int io_uring_register_wait_reg(struct io_uring *ring,
|
469
490
|
struct io_uring_reg_wait *reg, int nr)
|
470
491
|
{
|
471
|
-
|
472
|
-
|
473
|
-
.struct_size = sizeof(*reg),
|
474
|
-
.nr_entries = nr,
|
475
|
-
.user_addr = (unsigned long) (uintptr_t) reg,
|
476
|
-
};
|
492
|
+
return -EINVAL;
|
493
|
+
}
|
477
494
|
|
478
|
-
|
495
|
+
int io_uring_register_region(struct io_uring *ring,
|
496
|
+
struct io_uring_mem_region_reg *reg)
|
497
|
+
{
|
498
|
+
return do_register(ring, IORING_REGISTER_MEM_REGION, reg, 1);
|
499
|
+
}
|
500
|
+
|
501
|
+
int io_uring_set_iowait(struct io_uring *ring, bool enable_iowait)
|
502
|
+
{
|
503
|
+
if (!(ring->features & IORING_FEAT_NO_IOWAIT))
|
504
|
+
return -EOPNOTSUPP;
|
505
|
+
if (enable_iowait)
|
506
|
+
ring->int_flags &= ~INT_FLAG_NO_IOWAIT;
|
507
|
+
else
|
508
|
+
ring->int_flags |= INT_FLAG_NO_IOWAIT;
|
509
|
+
return 0;
|
479
510
|
}
|
@@ -115,6 +115,10 @@ static inline void initialize_sanitize_handlers()
|
|
115
115
|
sanitize_handlers[IORING_OP_FTRUNCATE] = sanitize_sqe_addr;
|
116
116
|
sanitize_handlers[IORING_OP_BIND] = sanitize_sqe_addr;
|
117
117
|
sanitize_handlers[IORING_OP_LISTEN] = sanitize_sqe_addr;
|
118
|
+
sanitize_handlers[IORING_OP_RECV_ZC] = sanitize_sqe_addr,
|
119
|
+
sanitize_handlers[IORING_OP_EPOLL_WAIT] = sanitize_sqe_addr,
|
120
|
+
sanitize_handlers[IORING_OP_READV_FIXED] = sanitize_sqe_addr,
|
121
|
+
sanitize_handlers[IORING_OP_WRITEV_FIXED] = sanitize_sqe_addr,
|
118
122
|
sanitize_handlers_initialized = true;
|
119
123
|
}
|
120
124
|
|
@@ -122,18 +126,11 @@ void liburing_sanitize_ring(struct io_uring *ring)
|
|
122
126
|
{
|
123
127
|
struct io_uring_sq *sq = &ring->sq;
|
124
128
|
struct io_uring_sqe *sqe;
|
125
|
-
unsigned
|
126
|
-
|
129
|
+
unsigned head = io_uring_load_sq_head(ring);
|
130
|
+
unsigned shift = io_uring_sqe_shift(ring);
|
127
131
|
|
128
132
|
initialize_sanitize_handlers();
|
129
133
|
|
130
|
-
if (ring->flags & IORING_SETUP_SQE128)
|
131
|
-
shift = 1;
|
132
|
-
if (!(ring->flags & IORING_SETUP_SQPOLL))
|
133
|
-
head = *sq->khead;
|
134
|
-
else
|
135
|
-
head = io_uring_smp_load_acquire(sq->khead);
|
136
|
-
|
137
134
|
while (head != sq->sqe_tail) {
|
138
135
|
sqe = &sq->sqes[(head & sq->ring_mask) << shift];
|
139
136
|
if (sqe->opcode < IORING_OP_LAST)
|
data/vendor/liburing/src/setup.c
CHANGED
@@ -94,18 +94,25 @@ void io_uring_setup_ring_pointers(struct io_uring_params *p,
|
|
94
94
|
cq->ring_entries = *cq->kring_entries;
|
95
95
|
}
|
96
96
|
|
97
|
+
static size_t params_sqes_size(const struct io_uring_params *p, unsigned sqes)
|
98
|
+
{
|
99
|
+
sqes <<= io_uring_sqe_shift_from_flags(p->flags);
|
100
|
+
return sqes * sizeof(struct io_uring_sqe);
|
101
|
+
}
|
102
|
+
|
103
|
+
static size_t params_cq_size(const struct io_uring_params *p, unsigned cqes)
|
104
|
+
{
|
105
|
+
cqes <<= io_uring_cqe_shift_from_flags(p->flags);
|
106
|
+
return cqes * sizeof(struct io_uring_cqe);
|
107
|
+
}
|
108
|
+
|
97
109
|
int io_uring_mmap(int fd, struct io_uring_params *p, struct io_uring_sq *sq,
|
98
110
|
struct io_uring_cq *cq)
|
99
111
|
{
|
100
|
-
size_t size;
|
101
112
|
int ret;
|
102
113
|
|
103
|
-
size = sizeof(struct io_uring_cqe);
|
104
|
-
if (p->flags & IORING_SETUP_CQE32)
|
105
|
-
size += sizeof(struct io_uring_cqe);
|
106
|
-
|
107
114
|
sq->ring_sz = p->sq_off.array + p->sq_entries * sizeof(unsigned);
|
108
|
-
cq->ring_sz = p->cq_off.cqes + p->cq_entries
|
115
|
+
cq->ring_sz = p->cq_off.cqes + params_cq_size(p, p->cq_entries);
|
109
116
|
|
110
117
|
if (p->features & IORING_FEAT_SINGLE_MMAP) {
|
111
118
|
if (cq->ring_sz > sq->ring_sz)
|
@@ -131,11 +138,9 @@ int io_uring_mmap(int fd, struct io_uring_params *p, struct io_uring_sq *sq,
|
|
131
138
|
}
|
132
139
|
}
|
133
140
|
|
134
|
-
|
135
|
-
|
136
|
-
|
137
|
-
sq->sqes = __sys_mmap(0, size * p->sq_entries, PROT_READ | PROT_WRITE,
|
138
|
-
MAP_SHARED | MAP_POPULATE, fd, IORING_OFF_SQES);
|
141
|
+
sq->sqes = __sys_mmap(0, params_sqes_size(p, p->sq_entries),
|
142
|
+
PROT_READ | PROT_WRITE, MAP_SHARED | MAP_POPULATE,
|
143
|
+
fd, IORING_OFF_SQES);
|
139
144
|
if (IS_ERR(sq->sqes)) {
|
140
145
|
ret = PTR_ERR(sq->sqes);
|
141
146
|
err:
|
@@ -160,6 +165,12 @@ __cold int io_uring_queue_mmap(int fd, struct io_uring_params *p,
|
|
160
165
|
return io_uring_mmap(fd, p, &ring->sq, &ring->cq);
|
161
166
|
}
|
162
167
|
|
168
|
+
static size_t io_uring_sqes_size(const struct io_uring *ring)
|
169
|
+
{
|
170
|
+
return (ring->sq.ring_entries << io_uring_sqe_shift(ring)) *
|
171
|
+
sizeof(struct io_uring_sqe);
|
172
|
+
}
|
173
|
+
|
163
174
|
/*
|
164
175
|
* Ensure that the mmap'ed rings aren't available to a child after a fork(2).
|
165
176
|
* This uses madvise(..., MADV_DONTFORK) on the mmap'ed ranges.
|
@@ -172,10 +183,7 @@ __cold int io_uring_ring_dontfork(struct io_uring *ring)
|
|
172
183
|
if (!ring->sq.ring_ptr || !ring->sq.sqes || !ring->cq.ring_ptr)
|
173
184
|
return -EINVAL;
|
174
185
|
|
175
|
-
len =
|
176
|
-
if (ring->flags & IORING_SETUP_SQE128)
|
177
|
-
len += 64;
|
178
|
-
len *= ring->sq.ring_entries;
|
186
|
+
len = io_uring_sqes_size(ring);
|
179
187
|
ret = __sys_madvise(ring->sq.sqes, len, MADV_DONTFORK);
|
180
188
|
if (ret < 0)
|
181
189
|
return ret;
|
@@ -209,7 +217,7 @@ static int io_uring_alloc_huge(unsigned entries, struct io_uring_params *p,
|
|
209
217
|
{
|
210
218
|
unsigned long page_size = get_page_size();
|
211
219
|
unsigned sq_entries, cq_entries;
|
212
|
-
size_t ring_mem, sqes_mem
|
220
|
+
size_t ring_mem, sqes_mem;
|
213
221
|
unsigned long mem_used = 0;
|
214
222
|
void *ptr;
|
215
223
|
int ret;
|
@@ -220,15 +228,12 @@ static int io_uring_alloc_huge(unsigned entries, struct io_uring_params *p,
|
|
220
228
|
|
221
229
|
ring_mem = KRING_SIZE;
|
222
230
|
|
223
|
-
sqes_mem =
|
231
|
+
sqes_mem = params_sqes_size(p, sq_entries);
|
224
232
|
if (!(p->flags & IORING_SETUP_NO_SQARRAY))
|
225
233
|
sqes_mem += sq_entries * sizeof(unsigned);
|
226
234
|
sqes_mem = (sqes_mem + page_size - 1) & ~(page_size - 1);
|
227
235
|
|
228
|
-
|
229
|
-
if (p->flags & IORING_SETUP_CQE32)
|
230
|
-
cqes_mem *= 2;
|
231
|
-
ring_mem += sqes_mem + cqes_mem;
|
236
|
+
ring_mem += sqes_mem + params_cq_size(p, cq_entries);
|
232
237
|
mem_used = ring_mem;
|
233
238
|
mem_used = (mem_used + page_size - 1) & ~(page_size - 1);
|
234
239
|
|
@@ -359,6 +364,13 @@ int __io_uring_queue_init_params(unsigned entries, struct io_uring *ring,
|
|
359
364
|
} else {
|
360
365
|
ring->ring_fd = fd;
|
361
366
|
}
|
367
|
+
/*
|
368
|
+
* IOPOLL always needs to enter, except if SQPOLL is set as well.
|
369
|
+
* Use an internal flag to check for this.
|
370
|
+
*/
|
371
|
+
if ((ring->flags & (IORING_SETUP_IOPOLL|IORING_SETUP_SQPOLL)) ==
|
372
|
+
IORING_SETUP_IOPOLL)
|
373
|
+
ring->int_flags |= INT_FLAG_CQ_ENTER;
|
362
374
|
|
363
375
|
return ret;
|
364
376
|
}
|
@@ -430,20 +442,10 @@ __cold void io_uring_queue_exit(struct io_uring *ring)
|
|
430
442
|
{
|
431
443
|
struct io_uring_sq *sq = &ring->sq;
|
432
444
|
struct io_uring_cq *cq = &ring->cq;
|
433
|
-
size_t sqe_size;
|
434
445
|
|
435
|
-
if (!
|
436
|
-
|
437
|
-
if (ring->flags & IORING_SETUP_SQE128)
|
438
|
-
sqe_size += 64;
|
439
|
-
__sys_munmap(sq->sqes, sqe_size * sq->ring_entries);
|
446
|
+
if (!(ring->int_flags & INT_FLAG_APP_MEM)) {
|
447
|
+
__sys_munmap(sq->sqes, io_uring_sqes_size(ring));
|
440
448
|
io_uring_unmap_rings(sq, cq);
|
441
|
-
} else {
|
442
|
-
if (!(ring->int_flags & INT_FLAG_APP_MEM)) {
|
443
|
-
__sys_munmap(sq->sqes,
|
444
|
-
*sq->kring_entries * sizeof(struct io_uring_sqe));
|
445
|
-
io_uring_unmap_rings(sq, cq);
|
446
|
-
}
|
447
449
|
}
|
448
450
|
|
449
451
|
/*
|
@@ -508,18 +510,11 @@ static size_t rings_size(struct io_uring_params *p, unsigned entries,
|
|
508
510
|
{
|
509
511
|
size_t pages, sq_size, cq_size;
|
510
512
|
|
511
|
-
cq_size =
|
512
|
-
if (p->flags & IORING_SETUP_CQE32)
|
513
|
-
cq_size += sizeof(struct io_uring_cqe);
|
514
|
-
cq_size *= cq_entries;
|
515
|
-
cq_size += KRING_SIZE;
|
513
|
+
cq_size = KRING_SIZE + params_cq_size(p, cq_entries);
|
516
514
|
cq_size = (cq_size + 63) & ~63UL;
|
517
515
|
pages = (size_t) 1 << npages(cq_size, page_size);
|
518
516
|
|
519
|
-
sq_size =
|
520
|
-
if (p->flags & IORING_SETUP_SQE128)
|
521
|
-
sq_size += 64;
|
522
|
-
sq_size *= entries;
|
517
|
+
sq_size = params_sqes_size(p, entries);
|
523
518
|
pages += (size_t) 1 << npages(sq_size, page_size);
|
524
519
|
return pages * page_size;
|
525
520
|
}
|
@@ -686,32 +681,3 @@ int io_uring_free_buf_ring(struct io_uring *ring, struct io_uring_buf_ring *br,
|
|
686
681
|
__sys_munmap(br, nentries * sizeof(struct io_uring_buf));
|
687
682
|
return 0;
|
688
683
|
}
|
689
|
-
|
690
|
-
void io_uring_free_reg_wait(struct io_uring_reg_wait *reg, unsigned nentries)
|
691
|
-
{
|
692
|
-
__sys_munmap(reg, nentries * sizeof(struct io_uring_reg_wait));
|
693
|
-
}
|
694
|
-
|
695
|
-
struct io_uring_reg_wait *io_uring_setup_reg_wait(struct io_uring *ring,
|
696
|
-
unsigned nentries, int *err)
|
697
|
-
{
|
698
|
-
struct io_uring_reg_wait *reg;
|
699
|
-
size_t size = nentries * sizeof(*reg);
|
700
|
-
int ret;
|
701
|
-
|
702
|
-
reg = __sys_mmap(NULL, size, PROT_READ | PROT_WRITE,
|
703
|
-
MAP_SHARED|MAP_POPULATE|MAP_ANONYMOUS, -1, 0);
|
704
|
-
if (IS_ERR(reg)) {
|
705
|
-
*err = PTR_ERR(reg);
|
706
|
-
return NULL;
|
707
|
-
}
|
708
|
-
|
709
|
-
memset(reg, 0, size);
|
710
|
-
ret = io_uring_register_wait_reg(ring, reg, nentries);
|
711
|
-
if (!ret)
|
712
|
-
return reg;
|
713
|
-
|
714
|
-
__sys_munmap(reg, size);
|
715
|
-
*err = ret;
|
716
|
-
return NULL;
|
717
|
-
}
|
@@ -11,9 +11,9 @@ int io_uring_enter(unsigned int fd, unsigned int to_submit,
|
|
11
11
|
|
12
12
|
int io_uring_enter2(unsigned int fd, unsigned int to_submit,
|
13
13
|
unsigned int min_complete, unsigned int flags,
|
14
|
-
|
14
|
+
void *arg, size_t sz)
|
15
15
|
{
|
16
|
-
return __sys_io_uring_enter2(fd, to_submit, min_complete, flags,
|
16
|
+
return __sys_io_uring_enter2(fd, to_submit, min_complete, flags, arg,
|
17
17
|
sz);
|
18
18
|
}
|
19
19
|
|
@@ -81,6 +81,7 @@ test_srcs := \
|
|
81
81
|
eeed8b54e0df.c \
|
82
82
|
empty-eownerdead.c \
|
83
83
|
eploop.c \
|
84
|
+
epwait.c \
|
84
85
|
eventfd.c \
|
85
86
|
eventfd-disable.c \
|
86
87
|
eventfd-reg.c \
|
@@ -106,6 +107,7 @@ test_srcs := \
|
|
106
107
|
fixed-hugepage.c \
|
107
108
|
fixed-link.c \
|
108
109
|
fixed-reuse.c \
|
110
|
+
fixed-seg.c \
|
109
111
|
fpos.c \
|
110
112
|
fsnotify.c \
|
111
113
|
fsync.c \
|
@@ -121,6 +123,7 @@ test_srcs := \
|
|
121
123
|
io_uring_passthrough.c \
|
122
124
|
io_uring_register.c \
|
123
125
|
io_uring_setup.c \
|
126
|
+
iowait.c \
|
124
127
|
kallsyms.c \
|
125
128
|
lfs-openat.c \
|
126
129
|
lfs-openat-write.c \
|
@@ -165,10 +168,12 @@ test_srcs := \
|
|
165
168
|
pollfree.c \
|
166
169
|
probe.c \
|
167
170
|
read-before-exit.c \
|
171
|
+
read-inc-file.c \
|
168
172
|
read-mshot.c \
|
169
173
|
read-mshot-empty.c \
|
170
174
|
read-mshot-stdin.c \
|
171
175
|
read-write.c \
|
176
|
+
recv-bundle-short-ooo.c \
|
172
177
|
recv-msgall.c \
|
173
178
|
recv-msgall-stream.c \
|
174
179
|
recv-multishot.c \
|
@@ -193,6 +198,7 @@ test_srcs := \
|
|
193
198
|
send_recv.c \
|
194
199
|
send_recvmsg.c \
|
195
200
|
send-zerocopy.c \
|
201
|
+
sendmsg_iov_clean.c \
|
196
202
|
shared-wq.c \
|
197
203
|
short-read.c \
|
198
204
|
shutdown.c \
|
@@ -202,6 +208,7 @@ test_srcs := \
|
|
202
208
|
socket.c \
|
203
209
|
socket-io-cmd.c \
|
204
210
|
socket-getsetsock-cmd.c \
|
211
|
+
socket-nb.c \
|
205
212
|
socket-rw.c \
|
206
213
|
socket-rw-eagain.c \
|
207
214
|
socket-rw-offset.c \
|
@@ -237,6 +244,8 @@ test_srcs := \
|
|
237
244
|
wakeup-hang.c \
|
238
245
|
wq-aff.c \
|
239
246
|
xattr.c \
|
247
|
+
zcrx.c \
|
248
|
+
vec-regbuf.c \
|
240
249
|
# EOL
|
241
250
|
|
242
251
|
# Please keep this list sorted alphabetically.
|
@@ -199,7 +199,7 @@ static int basic_cmd_test(struct io_uring *ring, int op)
|
|
199
199
|
|
200
200
|
fd = open(filename, O_DIRECT | O_RDWR | O_EXCL);
|
201
201
|
if (fd < 0) {
|
202
|
-
if (errno ==
|
202
|
+
if (errno == EINVAL || errno == EBUSY)
|
203
203
|
return T_EXIT_SKIP;
|
204
204
|
fprintf(stderr, "open failed %i\n", errno);
|
205
205
|
return T_EXIT_FAIL;
|
@@ -248,6 +248,8 @@ static int test_fail_edge_cases(struct io_uring *ring, int op)
|
|
248
248
|
|
249
249
|
fd = open(filename, O_DIRECT | O_RDWR | O_EXCL);
|
250
250
|
if (fd < 0) {
|
251
|
+
if (errno == EINVAL || errno == EBUSY)
|
252
|
+
return T_EXIT_SKIP;
|
251
253
|
fprintf(stderr, "open failed %i\n", errno);
|
252
254
|
return T_EXIT_FAIL;
|
253
255
|
}
|
@@ -302,6 +304,8 @@ static int test_rdonly(struct io_uring *ring, int op)
|
|
302
304
|
|
303
305
|
fd = open(filename, O_DIRECT | O_RDONLY | O_EXCL);
|
304
306
|
if (fd < 0) {
|
307
|
+
if (errno == EINVAL || errno == EBUSY)
|
308
|
+
return T_EXIT_SKIP;
|
305
309
|
fprintf(stderr, "open failed %i\n", errno);
|
306
310
|
return T_EXIT_FAIL;
|
307
311
|
}
|
@@ -315,6 +319,8 @@ static int test_rdonly(struct io_uring *ring, int op)
|
|
315
319
|
|
316
320
|
fd = open(filename, O_DIRECT | O_RDWR | O_EXCL);
|
317
321
|
if (fd < 0) {
|
322
|
+
if (errno == EINVAL || errno == EBUSY)
|
323
|
+
return T_EXIT_SKIP;
|
318
324
|
fprintf(stderr, "open failed %i\n", errno);
|
319
325
|
return T_EXIT_FAIL;
|
320
326
|
}
|
@@ -354,6 +360,8 @@ int main(int argc, char *argv[])
|
|
354
360
|
|
355
361
|
fd = open(filename, O_DIRECT | O_RDONLY | O_EXCL);
|
356
362
|
if (fd < 0) {
|
363
|
+
if (errno == EINVAL || errno == EBUSY)
|
364
|
+
return T_EXIT_SKIP;
|
357
365
|
fprintf(stderr, "open failed %i\n", errno);
|
358
366
|
return T_EXIT_FAIL;
|
359
367
|
}
|
@@ -396,29 +404,29 @@ int main(int argc, char *argv[])
|
|
396
404
|
if (!opcodes[cmd_op].test)
|
397
405
|
continue;
|
398
406
|
ret = basic_cmd_test(&ring, cmd_op);
|
399
|
-
if (ret) {
|
400
|
-
if (ret == T_EXIT_SKIP)
|
401
|
-
continue;
|
402
|
-
|
407
|
+
if (ret == T_EXIT_FAIL) {
|
403
408
|
fprintf(stderr, "basic_cmd_test() failed, cmd %i\n",
|
404
409
|
cmd_op);
|
405
410
|
return T_EXIT_FAIL;
|
406
411
|
}
|
407
412
|
|
408
413
|
ret = test_rdonly(&ring, cmd_op);
|
409
|
-
if (ret) {
|
414
|
+
if (ret == T_EXIT_FAIL) {
|
410
415
|
fprintf(stderr, "test_rdonly() failed, cmd %i\n",
|
411
416
|
cmd_op);
|
412
417
|
return T_EXIT_FAIL;
|
413
418
|
}
|
414
419
|
|
415
420
|
ret = test_fail_edge_cases(&ring, cmd_op);
|
416
|
-
if (ret) {
|
421
|
+
if (ret == T_EXIT_FAIL) {
|
417
422
|
fprintf(stderr, "test_fail_edge_cases() failed, cmd %i\n",
|
418
423
|
cmd_op);
|
419
424
|
return T_EXIT_FAIL;
|
420
425
|
}
|
421
|
-
|
426
|
+
if (ret == T_EXIT_SKIP)
|
427
|
+
fret = T_EXIT_SKIP;
|
428
|
+
else
|
429
|
+
fret = T_EXIT_PASS;
|
422
430
|
}
|
423
431
|
|
424
432
|
io_uring_queue_exit(&ring);
|
@@ -371,8 +371,10 @@ static int test(int flags)
|
|
371
371
|
int ret;
|
372
372
|
|
373
373
|
ret = io_uring_queue_init(8, &ring, flags);
|
374
|
+
if (ret == -EINVAL)
|
375
|
+
return T_EXIT_SKIP;
|
374
376
|
if (ret) {
|
375
|
-
fprintf(stderr, "
|
377
|
+
fprintf(stderr, "io_uring_queue_init() = %d\n", ret);
|
376
378
|
return T_EXIT_FAIL;
|
377
379
|
}
|
378
380
|
|
@@ -413,30 +415,32 @@ static int test(int flags)
|
|
413
415
|
|
414
416
|
int main(int argc, char *argv[])
|
415
417
|
{
|
418
|
+
bool any_passed = false;
|
416
419
|
int ret;
|
417
420
|
|
418
421
|
if (argc > 1)
|
419
422
|
return T_EXIT_SKIP;
|
420
423
|
|
421
424
|
ret = test(0);
|
422
|
-
|
425
|
+
any_passed |= ret == T_EXIT_PASS;
|
426
|
+
if (ret == T_EXIT_FAIL) {
|
423
427
|
fprintf(stderr, "test 0 failed\n");
|
424
428
|
return T_EXIT_FAIL;
|
425
429
|
}
|
426
|
-
if (no_connect)
|
427
|
-
return T_EXIT_SKIP;
|
428
430
|
|
429
431
|
ret = test(IORING_SETUP_SQPOLL);
|
430
|
-
|
432
|
+
any_passed |= ret == T_EXIT_PASS;
|
433
|
+
if (ret == T_EXIT_FAIL) {
|
431
434
|
fprintf(stderr, "test SQPOLL failed\n");
|
432
435
|
return T_EXIT_FAIL;
|
433
436
|
}
|
434
437
|
|
435
438
|
ret = test(IORING_SETUP_SINGLE_ISSUER|IORING_SETUP_DEFER_TASKRUN);
|
436
|
-
|
439
|
+
any_passed |= ret == T_EXIT_PASS;
|
440
|
+
if (ret == T_EXIT_FAIL) {
|
437
441
|
fprintf(stderr, "test DEFER failed\n");
|
438
442
|
return T_EXIT_FAIL;
|
439
443
|
}
|
440
444
|
|
441
|
-
return T_EXIT_PASS;
|
445
|
+
return any_passed ? T_EXIT_PASS : T_EXIT_SKIP;
|
442
446
|
}
|