fast_curl 0.2.0 → 0.3.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +49 -12
- data/ext/fast_curl/extconf.rb +4 -0
- data/ext/fast_curl/fast_curl.c +859 -475
- data/lib/fast_curl/version.rb +1 -1
- metadata +36 -26
data/ext/fast_curl/fast_curl.c
CHANGED
|
@@ -4,14 +4,32 @@
|
|
|
4
4
|
#ifdef HAVE_RUBY_FIBER_SCHEDULER_H
|
|
5
5
|
#include <ruby/fiber/scheduler.h>
|
|
6
6
|
#endif
|
|
7
|
+
|
|
8
|
+
#if defined(HAVE_RUBY_FIBER_SCHEDULER_H) && defined(HAVE_RB_FIBER_SCHEDULER_CURRENT) && \
|
|
9
|
+
defined(HAVE_RB_FIBER_SCHEDULER_BLOCK) && defined(HAVE_RB_FIBER_SCHEDULER_UNBLOCK) && \
|
|
10
|
+
defined(HAVE_RB_FIBER_CURRENT)
|
|
11
|
+
#define FAST_CURL_HAVE_FIBER_SCHEDULER 1
|
|
12
|
+
#endif
|
|
7
13
|
#include <curl/curl.h>
|
|
14
|
+
#include <limits.h>
|
|
8
15
|
#include <stdlib.h>
|
|
9
16
|
#include <string.h>
|
|
17
|
+
#include <strings.h>
|
|
18
|
+
#include <time.h>
|
|
19
|
+
|
|
20
|
+
#if defined(__GNUC__) || defined(__clang__)
|
|
21
|
+
#define FAST_CURL_NORETURN __attribute__((noreturn))
|
|
22
|
+
#else
|
|
23
|
+
#define FAST_CURL_NORETURN
|
|
24
|
+
#endif
|
|
10
25
|
|
|
11
26
|
#define MAX_RESPONSE_SIZE (100 * 1024 * 1024)
|
|
12
27
|
#define MAX_REDIRECTS 5
|
|
13
28
|
#define MAX_TIMEOUT 300
|
|
14
29
|
#define MAX_RETRIES 10
|
|
30
|
+
#define MAX_REQUESTS 10000
|
|
31
|
+
#define MAX_CONNECTIONS 100
|
|
32
|
+
#define MAX_RETRY_DELAY_MS 30000
|
|
15
33
|
#define DEFAULT_RETRIES 1
|
|
16
34
|
#define DEFAULT_RETRY_DELAY 0
|
|
17
35
|
#define INITIAL_BUF_CAP 8192
|
|
@@ -26,31 +44,31 @@ static const CURLcode DEFAULT_RETRYABLE_CURLE[] = {
|
|
|
26
44
|
#define DEFAULT_RETRYABLE_CURLE_COUNT \
|
|
27
45
|
(int)(sizeof(DEFAULT_RETRYABLE_CURLE) / sizeof(DEFAULT_RETRYABLE_CURLE[0]))
|
|
28
46
|
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
static
|
|
46
|
-
static VALUE
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
static
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
47
|
+
typedef enum {
|
|
48
|
+
KEY_STATUS,
|
|
49
|
+
KEY_HEADERS,
|
|
50
|
+
KEY_BODY,
|
|
51
|
+
KEY_ERROR_CODE,
|
|
52
|
+
KEY_URL,
|
|
53
|
+
KEY_METHOD,
|
|
54
|
+
KEY_TIMEOUT,
|
|
55
|
+
KEY_CONNECTIONS,
|
|
56
|
+
KEY_COUNT_OPT,
|
|
57
|
+
KEY_RETRIES,
|
|
58
|
+
KEY_RETRY_DELAY,
|
|
59
|
+
KEY_RETRY_CODES,
|
|
60
|
+
KEY_LAST
|
|
61
|
+
} key_id_t;
|
|
62
|
+
|
|
63
|
+
static ID fast_ids[KEY_LAST];
|
|
64
|
+
static VALUE fast_syms[KEY_LAST];
|
|
65
|
+
|
|
66
|
+
#define SYM(key) fast_syms[key]
|
|
67
|
+
|
|
68
|
+
static const char *const KEY_NAMES[KEY_LAST] = {
|
|
69
|
+
"status", "headers", "body", "error_code", "url", "method",
|
|
70
|
+
"timeout", "connections", "count", "retries", "retry_delay", "retry_codes",
|
|
71
|
+
};
|
|
54
72
|
|
|
55
73
|
typedef struct {
|
|
56
74
|
char *data;
|
|
@@ -83,17 +101,22 @@ static size_t write_callback(char *ptr, size_t size, size_t nmemb, void *userdat
|
|
|
83
101
|
buffer_t *buf = (buffer_t *)userdata;
|
|
84
102
|
size_t total = size * nmemb;
|
|
85
103
|
|
|
86
|
-
if (
|
|
104
|
+
if (nmemb != 0 && total / nmemb != size)
|
|
105
|
+
return 0;
|
|
106
|
+
if (total > buf->max_size || buf->len > buf->max_size - total)
|
|
87
107
|
return 0;
|
|
88
|
-
}
|
|
89
108
|
|
|
90
109
|
if (buf->len + total >= buf->cap) {
|
|
91
110
|
size_t new_cap = (buf->cap == 0) ? INITIAL_BUF_CAP : buf->cap;
|
|
92
|
-
while (new_cap <= buf->len + total)
|
|
111
|
+
while (new_cap <= buf->len + total) {
|
|
112
|
+
if (new_cap > buf->max_size / 2) {
|
|
113
|
+
new_cap = buf->max_size;
|
|
114
|
+
break;
|
|
115
|
+
}
|
|
93
116
|
new_cap *= 2;
|
|
94
|
-
|
|
95
|
-
if (new_cap
|
|
96
|
-
|
|
117
|
+
}
|
|
118
|
+
if (new_cap < buf->len + total)
|
|
119
|
+
return 0;
|
|
97
120
|
|
|
98
121
|
char *new_data = realloc(buf->data, new_cap);
|
|
99
122
|
if (!new_data)
|
|
@@ -143,22 +166,30 @@ static size_t header_callback(char *ptr, size_t size, size_t nmemb, void *userda
|
|
|
143
166
|
header_list_t *h = (header_list_t *)userdata;
|
|
144
167
|
size_t total = size * nmemb;
|
|
145
168
|
|
|
146
|
-
if (total
|
|
169
|
+
if (nmemb != 0 && total / nmemb != size)
|
|
170
|
+
return 0;
|
|
171
|
+
|
|
172
|
+
size_t stripped = total;
|
|
173
|
+
while (stripped > 0 && (ptr[stripped - 1] == '\r' || ptr[stripped - 1] == '\n'))
|
|
174
|
+
stripped--;
|
|
175
|
+
|
|
176
|
+
if (stripped >= 5 && memcmp(ptr, "HTTP/", 5) == 0) {
|
|
177
|
+
header_list_reset(h);
|
|
178
|
+
return total;
|
|
179
|
+
}
|
|
180
|
+
|
|
181
|
+
if (stripped == 0)
|
|
147
182
|
return total;
|
|
148
183
|
|
|
149
184
|
if (h->count >= h->cap) {
|
|
150
185
|
int new_cap = (h->cap == 0) ? INITIAL_HEADER_CAP : h->cap * 2;
|
|
151
|
-
header_entry_t *
|
|
152
|
-
if (!
|
|
186
|
+
header_entry_t *ne = realloc(h->entries, sizeof(header_entry_t) * new_cap);
|
|
187
|
+
if (!ne)
|
|
153
188
|
return 0;
|
|
154
|
-
h->entries =
|
|
189
|
+
h->entries = ne;
|
|
155
190
|
h->cap = new_cap;
|
|
156
191
|
}
|
|
157
192
|
|
|
158
|
-
size_t stripped = total;
|
|
159
|
-
while (stripped > 0 && (ptr[stripped - 1] == '\r' || ptr[stripped - 1] == '\n'))
|
|
160
|
-
stripped--;
|
|
161
|
-
|
|
162
193
|
char *entry = malloc(stripped + 1);
|
|
163
194
|
if (!entry)
|
|
164
195
|
return 0;
|
|
@@ -168,8 +199,7 @@ static size_t header_callback(char *ptr, size_t size, size_t nmemb, void *userda
|
|
|
168
199
|
h->entries[h->count].str = entry;
|
|
169
200
|
h->entries[h->count].len = stripped;
|
|
170
201
|
h->count++;
|
|
171
|
-
|
|
172
|
-
return size * nmemb;
|
|
202
|
+
return total;
|
|
173
203
|
}
|
|
174
204
|
|
|
175
205
|
typedef struct {
|
|
@@ -179,17 +209,19 @@ typedef struct {
|
|
|
179
209
|
header_list_t headers;
|
|
180
210
|
struct curl_slist *req_headers;
|
|
181
211
|
int done;
|
|
212
|
+
int active;
|
|
182
213
|
CURLcode curl_result;
|
|
183
214
|
long http_status;
|
|
184
215
|
} request_ctx_t;
|
|
185
216
|
|
|
186
217
|
static inline void request_ctx_init(request_ctx_t *ctx, int index) {
|
|
187
|
-
ctx->easy =
|
|
218
|
+
ctx->easy = NULL;
|
|
188
219
|
ctx->index = index;
|
|
189
220
|
buffer_init(&ctx->body);
|
|
190
221
|
header_list_init(&ctx->headers);
|
|
191
222
|
ctx->req_headers = NULL;
|
|
192
223
|
ctx->done = 0;
|
|
224
|
+
ctx->active = 0;
|
|
193
225
|
ctx->curl_result = CURLE_OK;
|
|
194
226
|
ctx->http_status = 0;
|
|
195
227
|
}
|
|
@@ -205,6 +237,16 @@ static void request_ctx_free(request_ctx_t *ctx) {
|
|
|
205
237
|
curl_slist_free_all(ctx->req_headers);
|
|
206
238
|
ctx->req_headers = NULL;
|
|
207
239
|
}
|
|
240
|
+
ctx->active = 0;
|
|
241
|
+
}
|
|
242
|
+
|
|
243
|
+
static int request_ctx_prepare_easy(request_ctx_t *ctx) {
|
|
244
|
+
if (!ctx->easy) {
|
|
245
|
+
ctx->easy = curl_easy_init();
|
|
246
|
+
if (!ctx->easy)
|
|
247
|
+
return 0;
|
|
248
|
+
}
|
|
249
|
+
return 1;
|
|
208
250
|
}
|
|
209
251
|
|
|
210
252
|
static int request_ctx_reset_for_retry(request_ctx_t *ctx) {
|
|
@@ -222,6 +264,7 @@ static int request_ctx_reset_for_retry(request_ctx_t *ctx) {
|
|
|
222
264
|
if (!ctx->easy)
|
|
223
265
|
return 0;
|
|
224
266
|
ctx->done = 0;
|
|
267
|
+
ctx->active = 0;
|
|
225
268
|
ctx->curl_result = CURLE_OK;
|
|
226
269
|
ctx->http_status = 0;
|
|
227
270
|
return 1;
|
|
@@ -234,15 +277,134 @@ typedef struct {
|
|
|
234
277
|
int still_running;
|
|
235
278
|
long timeout_ms;
|
|
236
279
|
int max_connections;
|
|
280
|
+
volatile int cancelled;
|
|
281
|
+
|
|
282
|
+
int active_count;
|
|
283
|
+
int pending_pos;
|
|
284
|
+
int pending_count;
|
|
285
|
+
int *pending_indices;
|
|
237
286
|
} multi_session_t;
|
|
238
287
|
|
|
239
288
|
typedef struct {
|
|
240
289
|
int max_retries;
|
|
290
|
+
int retries_explicit;
|
|
241
291
|
long retry_delay_ms;
|
|
242
292
|
int *retry_http_codes;
|
|
243
293
|
int retry_http_count;
|
|
244
294
|
} retry_config_t;
|
|
245
295
|
|
|
296
|
+
static int contains_header_injection(const char *str, long len) {
|
|
297
|
+
for (long i = 0; i < len; i++) {
|
|
298
|
+
if (str[i] == '\r' || str[i] == '\n' || str[i] == '\0')
|
|
299
|
+
return 1;
|
|
300
|
+
}
|
|
301
|
+
return 0;
|
|
302
|
+
}
|
|
303
|
+
|
|
304
|
+
static int is_header_token_char(unsigned char c) {
|
|
305
|
+
if ((c >= 'A' && c <= 'Z') || (c >= 'a' && c <= 'z') || (c >= '0' && c <= '9'))
|
|
306
|
+
return 1;
|
|
307
|
+
switch (c) {
|
|
308
|
+
case '!':
|
|
309
|
+
case '#':
|
|
310
|
+
case '$':
|
|
311
|
+
case '%':
|
|
312
|
+
case '&':
|
|
313
|
+
case '\'':
|
|
314
|
+
case '*':
|
|
315
|
+
case '+':
|
|
316
|
+
case '-':
|
|
317
|
+
case '.':
|
|
318
|
+
case '^':
|
|
319
|
+
case '_':
|
|
320
|
+
case '`':
|
|
321
|
+
case '|':
|
|
322
|
+
case '~':
|
|
323
|
+
return 1;
|
|
324
|
+
default:
|
|
325
|
+
return 0;
|
|
326
|
+
}
|
|
327
|
+
}
|
|
328
|
+
|
|
329
|
+
static int is_valid_header_name(const char *str, long len) {
|
|
330
|
+
if (len <= 0)
|
|
331
|
+
return 0;
|
|
332
|
+
for (long i = 0; i < len; i++) {
|
|
333
|
+
if (!is_header_token_char((unsigned char)str[i]))
|
|
334
|
+
return 0;
|
|
335
|
+
}
|
|
336
|
+
return 1;
|
|
337
|
+
}
|
|
338
|
+
|
|
339
|
+
static VALUE hash_aref_symbol_or_string(VALUE hash, VALUE sym, ID id) {
|
|
340
|
+
VALUE value = rb_hash_aref(hash, sym);
|
|
341
|
+
if (!NIL_P(value))
|
|
342
|
+
return value;
|
|
343
|
+
|
|
344
|
+
const char *name = rb_id2name(id);
|
|
345
|
+
return name ? rb_hash_aref(hash, rb_str_new_cstr(name)) : Qnil;
|
|
346
|
+
}
|
|
347
|
+
|
|
348
|
+
static inline VALUE hash_aref_key(VALUE hash, key_id_t key) {
|
|
349
|
+
return hash_aref_symbol_or_string(hash, fast_syms[key], fast_ids[key]);
|
|
350
|
+
}
|
|
351
|
+
|
|
352
|
+
#ifdef FAST_CURL_HAVE_FIBER_SCHEDULER
|
|
353
|
+
static VALUE current_fiber_scheduler(void) {
|
|
354
|
+
VALUE sched = rb_fiber_scheduler_current();
|
|
355
|
+
if (sched == Qnil || sched == Qfalse)
|
|
356
|
+
return Qnil;
|
|
357
|
+
return sched;
|
|
358
|
+
}
|
|
359
|
+
|
|
360
|
+
typedef struct {
|
|
361
|
+
void *(*func)(void *);
|
|
362
|
+
void *arg;
|
|
363
|
+
VALUE scheduler;
|
|
364
|
+
VALUE blocker;
|
|
365
|
+
VALUE fiber;
|
|
366
|
+
} fiber_worker_ctx_t;
|
|
367
|
+
|
|
368
|
+
static void *fiber_worker_nogvl(void *arg) {
|
|
369
|
+
fiber_worker_ctx_t *c = (fiber_worker_ctx_t *)arg;
|
|
370
|
+
c->func(c->arg);
|
|
371
|
+
return NULL;
|
|
372
|
+
}
|
|
373
|
+
|
|
374
|
+
static VALUE fiber_worker_thread(void *arg) {
|
|
375
|
+
fiber_worker_ctx_t *c = (fiber_worker_ctx_t *)arg;
|
|
376
|
+
rb_thread_call_without_gvl(fiber_worker_nogvl, c, RUBY_UBF_PROCESS, NULL);
|
|
377
|
+
rb_fiber_scheduler_unblock(c->scheduler, c->blocker, c->fiber);
|
|
378
|
+
return Qnil;
|
|
379
|
+
}
|
|
380
|
+
|
|
381
|
+
static void run_via_fiber_worker(VALUE scheduler, void *(*func)(void *), void *arg) {
|
|
382
|
+
fiber_worker_ctx_t ctx = {
|
|
383
|
+
.func = func,
|
|
384
|
+
.arg = arg,
|
|
385
|
+
.scheduler = scheduler,
|
|
386
|
+
.blocker = rb_obj_alloc(rb_cObject),
|
|
387
|
+
.fiber = rb_fiber_current(),
|
|
388
|
+
};
|
|
389
|
+
VALUE th = rb_thread_create(fiber_worker_thread, &ctx);
|
|
390
|
+
rb_fiber_scheduler_block(scheduler, ctx.blocker, Qnil);
|
|
391
|
+
rb_funcall(th, rb_intern("join"), 0);
|
|
392
|
+
}
|
|
393
|
+
#endif
|
|
394
|
+
|
|
395
|
+
static void headers_hash_store(VALUE headers_hash, VALUE key, VALUE val) {
|
|
396
|
+
VALUE existing = rb_hash_aref(headers_hash, key);
|
|
397
|
+
|
|
398
|
+
if (NIL_P(existing)) {
|
|
399
|
+
rb_hash_aset(headers_hash, key, val);
|
|
400
|
+
} else if (RB_TYPE_P(existing, T_ARRAY)) {
|
|
401
|
+
rb_ary_push(existing, val);
|
|
402
|
+
} else {
|
|
403
|
+
VALUE values = rb_ary_new_from_args(2, existing, val);
|
|
404
|
+
rb_hash_aset(headers_hash, key, values);
|
|
405
|
+
}
|
|
406
|
+
}
|
|
407
|
+
|
|
246
408
|
static VALUE build_response(request_ctx_t *ctx) {
|
|
247
409
|
long status = 0;
|
|
248
410
|
curl_easy_getinfo(ctx->easy, CURLINFO_RESPONSE_CODE, &status);
|
|
@@ -251,77 +413,65 @@ static VALUE build_response(request_ctx_t *ctx) {
|
|
|
251
413
|
for (int i = 0; i < ctx->headers.count; i++) {
|
|
252
414
|
const char *hdr = ctx->headers.entries[i].str;
|
|
253
415
|
size_t hdr_len = ctx->headers.entries[i].len;
|
|
254
|
-
|
|
255
416
|
const char *colon = memchr(hdr, ':', hdr_len);
|
|
256
417
|
if (!colon)
|
|
257
418
|
continue;
|
|
258
419
|
|
|
259
420
|
VALUE key = rb_str_new(hdr, colon - hdr);
|
|
421
|
+
const char *vs = colon + 1;
|
|
422
|
+
const char *ve = hdr + hdr_len;
|
|
260
423
|
|
|
261
|
-
|
|
262
|
-
|
|
263
|
-
|
|
264
|
-
|
|
265
|
-
val_start++;
|
|
424
|
+
while (vs < ve && (*vs == ' ' || *vs == '\t'))
|
|
425
|
+
vs++;
|
|
426
|
+
while (ve > vs && (*(ve - 1) == ' ' || *(ve - 1) == '\t'))
|
|
427
|
+
ve--;
|
|
266
428
|
|
|
267
|
-
|
|
268
|
-
|
|
269
|
-
|
|
270
|
-
VALUE val = rb_str_new(val_start, val_end - val_start);
|
|
271
|
-
rb_hash_aset(headers_hash, key, val);
|
|
429
|
+
VALUE val = rb_str_new(vs, ve - vs);
|
|
430
|
+
headers_hash_store(headers_hash, key, val);
|
|
272
431
|
}
|
|
273
432
|
|
|
274
433
|
VALUE body_str =
|
|
275
434
|
ctx->body.data ? rb_str_new(ctx->body.data, ctx->body.len) : rb_str_new_cstr("");
|
|
276
435
|
|
|
277
436
|
VALUE result = rb_hash_new();
|
|
278
|
-
rb_hash_aset(result,
|
|
279
|
-
rb_hash_aset(result,
|
|
280
|
-
rb_hash_aset(result,
|
|
281
|
-
|
|
437
|
+
rb_hash_aset(result, SYM(KEY_STATUS), LONG2NUM(status));
|
|
438
|
+
rb_hash_aset(result, SYM(KEY_HEADERS), headers_hash);
|
|
439
|
+
rb_hash_aset(result, SYM(KEY_BODY), body_str);
|
|
282
440
|
return result;
|
|
283
441
|
}
|
|
284
442
|
|
|
285
443
|
static VALUE build_error_response(const char *message) {
|
|
286
|
-
VALUE
|
|
287
|
-
rb_hash_aset(
|
|
288
|
-
rb_hash_aset(
|
|
289
|
-
rb_hash_aset(
|
|
290
|
-
return
|
|
444
|
+
VALUE r = rb_hash_new();
|
|
445
|
+
rb_hash_aset(r, SYM(KEY_STATUS), INT2NUM(0));
|
|
446
|
+
rb_hash_aset(r, SYM(KEY_HEADERS), Qnil);
|
|
447
|
+
rb_hash_aset(r, SYM(KEY_BODY), rb_str_new_cstr(message));
|
|
448
|
+
return r;
|
|
291
449
|
}
|
|
292
450
|
|
|
293
451
|
static VALUE build_error_response_with_code(const char *message, int error_code) {
|
|
294
|
-
VALUE
|
|
295
|
-
rb_hash_aset(
|
|
296
|
-
|
|
297
|
-
rb_hash_aset(result, sym_body, rb_str_new_cstr(message));
|
|
298
|
-
rb_hash_aset(result, sym_error_code, INT2NUM(error_code));
|
|
299
|
-
return result;
|
|
452
|
+
VALUE r = build_error_response(message);
|
|
453
|
+
rb_hash_aset(r, SYM(KEY_ERROR_CODE), INT2NUM(error_code));
|
|
454
|
+
return r;
|
|
300
455
|
}
|
|
301
456
|
|
|
302
457
|
static int is_valid_url(const char *url) {
|
|
303
458
|
if (!url)
|
|
304
459
|
return 0;
|
|
305
|
-
|
|
306
|
-
|
|
307
|
-
|
|
308
|
-
if (url_len < 8 || url_len > 2048)
|
|
460
|
+
size_t len = strlen(url);
|
|
461
|
+
if (len < 8 || len > 2048)
|
|
309
462
|
return 0;
|
|
310
|
-
|
|
311
463
|
if (strncmp(url, "https://", 8) == 0)
|
|
312
464
|
return 1;
|
|
313
|
-
if (
|
|
465
|
+
if (len >= 7 && strncmp(url, "http://", 7) == 0)
|
|
314
466
|
return 1;
|
|
315
|
-
|
|
316
467
|
return 0;
|
|
317
468
|
}
|
|
318
469
|
|
|
319
|
-
#define CURL_SETOPT_CHECK(handle, option, value)
|
|
320
|
-
do {
|
|
321
|
-
CURLcode
|
|
322
|
-
if (
|
|
323
|
-
return
|
|
324
|
-
} \
|
|
470
|
+
#define CURL_SETOPT_CHECK(handle, option, value) \
|
|
471
|
+
do { \
|
|
472
|
+
CURLcode _r = curl_easy_setopt(handle, option, value); \
|
|
473
|
+
if (_r != CURLE_OK) \
|
|
474
|
+
return _r; \
|
|
325
475
|
} while (0)
|
|
326
476
|
|
|
327
477
|
static CURLcode setup_basic_options(CURL *easy, const char *url_str, long timeout_sec,
|
|
@@ -338,120 +488,168 @@ static CURLcode setup_basic_options(CURL *easy, const char *url_str, long timeou
|
|
|
338
488
|
CURL_SETOPT_CHECK(easy, CURLOPT_ACCEPT_ENCODING, "");
|
|
339
489
|
CURL_SETOPT_CHECK(easy, CURLOPT_PRIVATE, (char *)ctx);
|
|
340
490
|
CURL_SETOPT_CHECK(easy, CURLOPT_HTTP_VERSION, CURL_HTTP_VERSION_2TLS);
|
|
341
|
-
|
|
342
491
|
return CURLE_OK;
|
|
343
492
|
}
|
|
344
493
|
|
|
345
494
|
static CURLcode setup_security_options(CURL *easy) {
|
|
346
495
|
CURL_SETOPT_CHECK(easy, CURLOPT_SSL_VERIFYPEER, 1L);
|
|
347
496
|
CURL_SETOPT_CHECK(easy, CURLOPT_SSL_VERIFYHOST, 2L);
|
|
348
|
-
|
|
349
|
-
#ifdef CURLOPT_PROTOCOLS_STR
|
|
497
|
+
#if LIBCURL_VERSION_NUM >= 0x075500
|
|
350
498
|
CURL_SETOPT_CHECK(easy, CURLOPT_PROTOCOLS_STR, "http,https");
|
|
351
499
|
CURL_SETOPT_CHECK(easy, CURLOPT_REDIR_PROTOCOLS_STR, "http,https");
|
|
352
500
|
#else
|
|
353
501
|
CURL_SETOPT_CHECK(easy, CURLOPT_PROTOCOLS, CURLPROTO_HTTP | CURLPROTO_HTTPS);
|
|
354
502
|
CURL_SETOPT_CHECK(easy, CURLOPT_REDIR_PROTOCOLS, CURLPROTO_HTTP | CURLPROTO_HTTPS);
|
|
355
503
|
#endif
|
|
356
|
-
|
|
357
504
|
return CURLE_OK;
|
|
358
505
|
}
|
|
359
506
|
|
|
360
|
-
static CURLcode
|
|
361
|
-
|
|
362
|
-
|
|
363
|
-
|
|
364
|
-
|
|
365
|
-
|
|
366
|
-
|
|
367
|
-
} else if (strcmp(m, "DELETE") == 0) {
|
|
368
|
-
CURL_SETOPT_CHECK(easy, CURLOPT_CUSTOMREQUEST, "DELETE");
|
|
369
|
-
} else if (strcmp(m, "PATCH") == 0) {
|
|
370
|
-
CURL_SETOPT_CHECK(easy, CURLOPT_CUSTOMREQUEST, "PATCH");
|
|
371
|
-
} else if (strcmp(m, "GET") != 0) {
|
|
372
|
-
CURL_SETOPT_CHECK(easy, CURLOPT_CUSTOMREQUEST, m);
|
|
373
|
-
}
|
|
374
|
-
}
|
|
507
|
+
static CURLcode set_body(CURL *easy, VALUE body) {
|
|
508
|
+
VALUE body_str = rb_String(body);
|
|
509
|
+
CURL_SETOPT_CHECK(easy, CURLOPT_POSTFIELDSIZE, (long)RSTRING_LEN(body_str));
|
|
510
|
+
CURL_SETOPT_CHECK(easy, CURLOPT_COPYPOSTFIELDS, StringValuePtr(body_str));
|
|
511
|
+
RB_GC_GUARD(body_str);
|
|
512
|
+
return CURLE_OK;
|
|
513
|
+
}
|
|
375
514
|
|
|
376
|
-
|
|
377
|
-
|
|
378
|
-
|
|
515
|
+
typedef struct {
|
|
516
|
+
const char *name;
|
|
517
|
+
const char *custom;
|
|
518
|
+
int post;
|
|
519
|
+
int nobody;
|
|
520
|
+
int allows_body;
|
|
521
|
+
} http_method_t;
|
|
522
|
+
|
|
523
|
+
static const http_method_t HTTP_METHODS[] = {
|
|
524
|
+
{"GET", NULL, 0, 0, 0}, {"POST", NULL, 1, 0, 1}, {"PUT", "PUT", 0, 0, 1},
|
|
525
|
+
{"DELETE", "DELETE", 0, 0, 1}, {"PATCH", "PATCH", 0, 0, 1}, {"HEAD", NULL, 0, 1, 0},
|
|
526
|
+
{"OPTIONS", "OPTIONS", 0, 0, 0},
|
|
527
|
+
};
|
|
528
|
+
|
|
529
|
+
static const http_method_t *find_http_method(const char *name) {
|
|
530
|
+
for (size_t i = 0; i < sizeof(HTTP_METHODS) / sizeof(HTTP_METHODS[0]); i++) {
|
|
531
|
+
if (strcasecmp(HTTP_METHODS[i].name, name) == 0)
|
|
532
|
+
return &HTTP_METHODS[i];
|
|
379
533
|
}
|
|
534
|
+
return NULL;
|
|
535
|
+
}
|
|
380
536
|
|
|
537
|
+
static CURLcode apply_http_method(CURL *easy, const http_method_t *method) {
|
|
538
|
+
if (method->post)
|
|
539
|
+
CURL_SETOPT_CHECK(easy, CURLOPT_POST, 1L);
|
|
540
|
+
if (method->custom)
|
|
541
|
+
CURL_SETOPT_CHECK(easy, CURLOPT_CUSTOMREQUEST, method->custom);
|
|
542
|
+
if (method->nobody)
|
|
543
|
+
CURL_SETOPT_CHECK(easy, CURLOPT_NOBODY, 1L);
|
|
381
544
|
return CURLE_OK;
|
|
382
545
|
}
|
|
383
546
|
|
|
384
|
-
static
|
|
385
|
-
|
|
547
|
+
static CURLcode setup_method_and_body(CURL *easy, VALUE method_value, VALUE body) {
|
|
548
|
+
const char *name = NIL_P(method_value) ? "GET" : StringValueCStr(method_value);
|
|
549
|
+
const http_method_t *method = find_http_method(name);
|
|
550
|
+
int has_body = !NIL_P(body);
|
|
386
551
|
|
|
387
|
-
|
|
388
|
-
|
|
389
|
-
|
|
552
|
+
if (!method)
|
|
553
|
+
rb_raise(rb_eArgError, "Unsupported HTTP method: %s", name);
|
|
554
|
+
if (has_body && !method->allows_body)
|
|
555
|
+
rb_raise(rb_eArgError, "%s requests must not include a body", method->name);
|
|
390
556
|
|
|
391
|
-
|
|
392
|
-
|
|
393
|
-
|
|
394
|
-
long need = klen + 2;
|
|
557
|
+
CURLcode res = apply_http_method(easy, method);
|
|
558
|
+
if (res != CURLE_OK)
|
|
559
|
+
return res;
|
|
395
560
|
|
|
396
|
-
|
|
397
|
-
|
|
398
|
-
if (
|
|
399
|
-
return
|
|
561
|
+
if (has_body) {
|
|
562
|
+
res = set_body(easy, body);
|
|
563
|
+
if (res != CURLE_OK)
|
|
564
|
+
return res;
|
|
565
|
+
}
|
|
400
566
|
|
|
401
|
-
|
|
402
|
-
|
|
403
|
-
|
|
567
|
+
RB_GC_GUARD(method_value);
|
|
568
|
+
RB_GC_GUARD(body);
|
|
569
|
+
return CURLE_OK;
|
|
570
|
+
}
|
|
404
571
|
|
|
405
|
-
|
|
572
|
+
static void append_request_header(request_ctx_t *ctx, const char *buf) {
|
|
573
|
+
struct curl_slist *new_headers = curl_slist_append(ctx->req_headers, buf);
|
|
574
|
+
if (!new_headers)
|
|
575
|
+
rb_raise(rb_eNoMemError, "failed to allocate request header");
|
|
576
|
+
ctx->req_headers = new_headers;
|
|
577
|
+
}
|
|
406
578
|
|
|
407
|
-
|
|
408
|
-
|
|
579
|
+
static char *alloc_header_line(const char *key, long key_len, const char *value, long value_len,
|
|
580
|
+
char stack_buf[HEADER_LINE_BUF_SIZE]) {
|
|
581
|
+
int has_value = value && value_len > 0;
|
|
582
|
+
long need = key_len + (has_value ? 2 + value_len : 1) + 1;
|
|
583
|
+
char *buf = need > HEADER_LINE_BUF_SIZE ? malloc((size_t)need) : stack_buf;
|
|
584
|
+
|
|
585
|
+
if (!buf)
|
|
586
|
+
rb_raise(rb_eNoMemError, "failed to allocate request header");
|
|
587
|
+
|
|
588
|
+
memcpy(buf, key, (size_t)key_len);
|
|
589
|
+
if (has_value) {
|
|
590
|
+
buf[key_len] = ':';
|
|
591
|
+
buf[key_len + 1] = ' ';
|
|
592
|
+
memcpy(buf + key_len + 2, value, (size_t)value_len);
|
|
593
|
+
buf[key_len + 2 + value_len] = '\0';
|
|
409
594
|
} else {
|
|
410
|
-
|
|
411
|
-
|
|
412
|
-
|
|
413
|
-
char stack_buf[HEADER_LINE_BUF_SIZE];
|
|
414
|
-
char *buf = stack_buf;
|
|
415
|
-
long need = klen + 2 + vlen + 1;
|
|
595
|
+
buf[key_len] = ';';
|
|
596
|
+
buf[key_len + 1] = '\0';
|
|
597
|
+
}
|
|
416
598
|
|
|
417
|
-
|
|
418
|
-
|
|
419
|
-
|
|
420
|
-
|
|
599
|
+
return buf;
|
|
600
|
+
}
|
|
601
|
+
|
|
602
|
+
static void append_formatted_header(request_ctx_t *ctx, const char *key, long key_len,
|
|
603
|
+
const char *value, long value_len) {
|
|
604
|
+
char stack_buf[HEADER_LINE_BUF_SIZE];
|
|
605
|
+
char *line = alloc_header_line(key, key_len, value, value_len, stack_buf);
|
|
606
|
+
append_request_header(ctx, line);
|
|
607
|
+
if (line != stack_buf)
|
|
608
|
+
free(line);
|
|
609
|
+
}
|
|
421
610
|
|
|
422
|
-
|
|
423
|
-
|
|
424
|
-
|
|
425
|
-
|
|
426
|
-
|
|
611
|
+
static int header_iter_cb(VALUE key, VALUE val, VALUE arg) {
|
|
612
|
+
request_ctx_t *ctx = (request_ctx_t *)arg;
|
|
613
|
+
VALUE key_str = rb_String(key);
|
|
614
|
+
VALUE val_str = NIL_P(val) ? Qnil : rb_String(val);
|
|
615
|
+
const char *k = RSTRING_PTR(key_str);
|
|
616
|
+
long klen = RSTRING_LEN(key_str);
|
|
617
|
+
const char *v = NULL;
|
|
618
|
+
long vlen = 0;
|
|
427
619
|
|
|
428
|
-
|
|
620
|
+
if (!is_valid_header_name(k, klen) || contains_header_injection(k, klen))
|
|
621
|
+
rb_raise(rb_eArgError, "Invalid HTTP header name");
|
|
429
622
|
|
|
430
|
-
|
|
431
|
-
|
|
623
|
+
if (!NIL_P(val_str) && RSTRING_LEN(val_str) > 0) {
|
|
624
|
+
v = RSTRING_PTR(val_str);
|
|
625
|
+
vlen = RSTRING_LEN(val_str);
|
|
626
|
+
if (contains_header_injection(v, vlen))
|
|
627
|
+
rb_raise(rb_eArgError, "Invalid HTTP header value");
|
|
432
628
|
}
|
|
433
629
|
|
|
630
|
+
append_formatted_header(ctx, k, klen, v, vlen);
|
|
631
|
+
|
|
632
|
+
RB_GC_GUARD(key_str);
|
|
633
|
+
RB_GC_GUARD(val_str);
|
|
434
634
|
return ST_CONTINUE;
|
|
435
635
|
}
|
|
436
636
|
|
|
437
637
|
static int setup_easy_handle(request_ctx_t *ctx, VALUE request, long timeout_sec) {
|
|
438
|
-
|
|
439
|
-
|
|
440
|
-
VALUE
|
|
441
|
-
VALUE
|
|
638
|
+
Check_Type(request, T_HASH);
|
|
639
|
+
|
|
640
|
+
VALUE url = hash_aref_key(request, KEY_URL);
|
|
641
|
+
VALUE method = hash_aref_key(request, KEY_METHOD);
|
|
642
|
+
VALUE headers = hash_aref_key(request, KEY_HEADERS);
|
|
643
|
+
VALUE body = hash_aref_key(request, KEY_BODY);
|
|
442
644
|
|
|
443
645
|
if (NIL_P(url))
|
|
444
646
|
return 0;
|
|
445
647
|
|
|
446
648
|
const char *url_str = StringValueCStr(url);
|
|
447
|
-
|
|
448
|
-
if (!is_valid_url(url_str)) {
|
|
649
|
+
if (!is_valid_url(url_str))
|
|
449
650
|
rb_raise(rb_eArgError, "Invalid URL: %s", url_str);
|
|
450
|
-
}
|
|
451
651
|
|
|
452
|
-
CURLcode res;
|
|
453
|
-
|
|
454
|
-
res = setup_basic_options(ctx->easy, url_str, timeout_sec, ctx);
|
|
652
|
+
CURLcode res = setup_basic_options(ctx->easy, url_str, timeout_sec, ctx);
|
|
455
653
|
if (res != CURLE_OK)
|
|
456
654
|
return 0;
|
|
457
655
|
|
|
@@ -463,9 +661,9 @@ static int setup_easy_handle(request_ctx_t *ctx, VALUE request, long timeout_sec
|
|
|
463
661
|
if (res != CURLE_OK)
|
|
464
662
|
return 0;
|
|
465
663
|
|
|
466
|
-
if (!NIL_P(headers)
|
|
664
|
+
if (!NIL_P(headers)) {
|
|
665
|
+
Check_Type(headers, T_HASH);
|
|
467
666
|
rb_hash_foreach(headers, header_iter_cb, (VALUE)ctx);
|
|
468
|
-
|
|
469
667
|
if (ctx->req_headers) {
|
|
470
668
|
res = curl_easy_setopt(ctx->easy, CURLOPT_HTTPHEADER, ctx->req_headers);
|
|
471
669
|
if (res != CURLE_OK)
|
|
@@ -473,46 +671,30 @@ static int setup_easy_handle(request_ctx_t *ctx, VALUE request, long timeout_sec
|
|
|
473
671
|
}
|
|
474
672
|
}
|
|
475
673
|
|
|
674
|
+
RB_GC_GUARD(url);
|
|
675
|
+
RB_GC_GUARD(method);
|
|
676
|
+
RB_GC_GUARD(headers);
|
|
677
|
+
RB_GC_GUARD(body);
|
|
678
|
+
RB_GC_GUARD(request);
|
|
476
679
|
return 1;
|
|
477
680
|
}
|
|
478
681
|
|
|
479
|
-
static void *perform_without_gvl(void *arg) {
|
|
480
|
-
multi_session_t *session = (multi_session_t *)arg;
|
|
481
|
-
|
|
482
|
-
while (session->still_running > 0) {
|
|
483
|
-
CURLMcode mc = curl_multi_perform(session->multi, &session->still_running);
|
|
484
|
-
if (mc != CURLM_OK)
|
|
485
|
-
break;
|
|
486
|
-
|
|
487
|
-
if (session->still_running > 0) {
|
|
488
|
-
int numfds = 0;
|
|
489
|
-
mc = curl_multi_poll(session->multi, NULL, 0, POLL_TIMEOUT_MS, &numfds);
|
|
490
|
-
if (mc != CURLM_OK)
|
|
491
|
-
break;
|
|
492
|
-
}
|
|
493
|
-
}
|
|
494
|
-
|
|
495
|
-
return NULL;
|
|
496
|
-
}
|
|
497
|
-
|
|
498
682
|
static void *poll_without_gvl(void *arg) {
|
|
499
|
-
multi_session_t *
|
|
683
|
+
multi_session_t *s = (multi_session_t *)arg;
|
|
684
|
+
if (s->cancelled)
|
|
685
|
+
return NULL;
|
|
686
|
+
|
|
500
687
|
int numfds = 0;
|
|
501
|
-
curl_multi_poll(
|
|
502
|
-
curl_multi_perform(
|
|
688
|
+
curl_multi_poll(s->multi, NULL, 0, POLL_TIMEOUT_MS, &numfds);
|
|
689
|
+
curl_multi_perform(s->multi, &s->still_running);
|
|
503
690
|
return NULL;
|
|
504
691
|
}
|
|
505
692
|
|
|
506
693
|
static void unblock_perform(void *arg) {
|
|
507
|
-
(
|
|
508
|
-
|
|
509
|
-
|
|
510
|
-
|
|
511
|
-
#ifdef HAVE_RB_FIBER_SCHEDULER_CURRENT
|
|
512
|
-
VALUE scheduler = rb_fiber_scheduler_current();
|
|
513
|
-
return scheduler != Qnil && scheduler != Qfalse;
|
|
514
|
-
#else
|
|
515
|
-
return 0;
|
|
694
|
+
multi_session_t *s = (multi_session_t *)arg;
|
|
695
|
+
s->cancelled = 1;
|
|
696
|
+
#ifdef HAVE_CURL_MULTI_WAKEUP
|
|
697
|
+
curl_multi_wakeup(s->multi);
|
|
516
698
|
#endif
|
|
517
699
|
}
|
|
518
700
|
|
|
@@ -523,6 +705,26 @@ typedef struct {
|
|
|
523
705
|
int stream;
|
|
524
706
|
} completion_ctx_t;
|
|
525
707
|
|
|
708
|
+
static VALUE build_result_pair(int index, VALUE response) {
|
|
709
|
+
return rb_ary_new_from_args(2, INT2NUM(index), response);
|
|
710
|
+
}
|
|
711
|
+
|
|
712
|
+
static int record_immediate_error(completion_ctx_t *cctx, int index, const char *message) {
|
|
713
|
+
if (cctx->stream || cctx->target > 0) {
|
|
714
|
+
VALUE pair = build_result_pair(index, build_error_response(message));
|
|
715
|
+
|
|
716
|
+
if (cctx->stream)
|
|
717
|
+
rb_yield(pair);
|
|
718
|
+
else
|
|
719
|
+
rb_ary_push(cctx->results, pair);
|
|
720
|
+
|
|
721
|
+
cctx->completed++;
|
|
722
|
+
if (cctx->target > 0 && cctx->completed >= cctx->target)
|
|
723
|
+
return 1;
|
|
724
|
+
}
|
|
725
|
+
return 0;
|
|
726
|
+
}
|
|
727
|
+
|
|
526
728
|
static int process_completed(multi_session_t *session, completion_ctx_t *cctx) {
|
|
527
729
|
CURLMsg *msg;
|
|
528
730
|
int msgs_left;
|
|
@@ -535,27 +737,33 @@ static int process_completed(multi_session_t *session, completion_ctx_t *cctx) {
|
|
|
535
737
|
curl_easy_getinfo(msg->easy_handle, CURLINFO_PRIVATE, (char **)&ctx);
|
|
536
738
|
if (!ctx || ctx->done)
|
|
537
739
|
continue;
|
|
740
|
+
|
|
741
|
+
if (ctx->active) {
|
|
742
|
+
curl_multi_remove_handle(session->multi, ctx->easy);
|
|
743
|
+
ctx->active = 0;
|
|
744
|
+
if (session->active_count > 0)
|
|
745
|
+
session->active_count--;
|
|
746
|
+
}
|
|
747
|
+
|
|
538
748
|
ctx->done = 1;
|
|
539
749
|
ctx->curl_result = msg->data.result;
|
|
540
|
-
if (msg->data.result == CURLE_OK)
|
|
750
|
+
if (msg->data.result == CURLE_OK)
|
|
541
751
|
curl_easy_getinfo(ctx->easy, CURLINFO_RESPONSE_CODE, &ctx->http_status);
|
|
542
|
-
}
|
|
543
752
|
|
|
544
|
-
if (cctx->stream) {
|
|
545
|
-
VALUE response
|
|
546
|
-
|
|
547
|
-
|
|
548
|
-
|
|
549
|
-
|
|
550
|
-
|
|
551
|
-
|
|
552
|
-
|
|
553
|
-
|
|
554
|
-
|
|
555
|
-
} else {
|
|
556
|
-
cctx->completed++;
|
|
753
|
+
if (cctx->stream || cctx->target > 0) {
|
|
754
|
+
VALUE response = (msg->data.result == CURLE_OK)
|
|
755
|
+
? build_response(ctx)
|
|
756
|
+
: build_error_response_with_code(
|
|
757
|
+
curl_easy_strerror(msg->data.result), (int)msg->data.result);
|
|
758
|
+
VALUE pair = build_result_pair(ctx->index, response);
|
|
759
|
+
|
|
760
|
+
if (cctx->stream)
|
|
761
|
+
rb_yield(pair);
|
|
762
|
+
else
|
|
763
|
+
rb_ary_push(cctx->results, pair);
|
|
557
764
|
}
|
|
558
765
|
|
|
766
|
+
cctx->completed++;
|
|
559
767
|
if (cctx->target > 0 && cctx->completed >= cctx->target)
|
|
560
768
|
return 1;
|
|
561
769
|
}
|
|
@@ -563,59 +771,121 @@ static int process_completed(multi_session_t *session, completion_ctx_t *cctx) {
|
|
|
563
771
|
return 0;
|
|
564
772
|
}
|
|
565
773
|
|
|
566
|
-
static
|
|
567
|
-
if (
|
|
568
|
-
|
|
569
|
-
CURLMcode mc = curl_multi_perform(session->multi, &session->still_running);
|
|
570
|
-
if (mc != CURLM_OK)
|
|
571
|
-
break;
|
|
572
|
-
if (process_completed(session, cctx))
|
|
573
|
-
break;
|
|
574
|
-
if (session->still_running == 0)
|
|
575
|
-
break;
|
|
774
|
+
static int next_pending_index(multi_session_t *session) {
|
|
775
|
+
if (session->pending_pos >= session->pending_count)
|
|
776
|
+
return -1;
|
|
576
777
|
|
|
577
|
-
|
|
578
|
-
|
|
579
|
-
|
|
580
|
-
|
|
581
|
-
|
|
582
|
-
|
|
583
|
-
|
|
584
|
-
|
|
585
|
-
|
|
586
|
-
|
|
587
|
-
|
|
588
|
-
|
|
589
|
-
|
|
590
|
-
|
|
591
|
-
|
|
592
|
-
|
|
593
|
-
|
|
594
|
-
|
|
595
|
-
|
|
778
|
+
if (session->pending_indices)
|
|
779
|
+
return session->pending_indices[session->pending_pos++];
|
|
780
|
+
|
|
781
|
+
return session->pending_pos++;
|
|
782
|
+
}
|
|
783
|
+
|
|
784
|
+
static int activate_request(multi_session_t *session, VALUE requests, int idx, int *invalid,
|
|
785
|
+
long timeout_sec) {
|
|
786
|
+
request_ctx_t *ctx = &session->requests[idx];
|
|
787
|
+
|
|
788
|
+
if (invalid[idx] || ctx->done)
|
|
789
|
+
return 0;
|
|
790
|
+
|
|
791
|
+
if (!request_ctx_prepare_easy(ctx))
|
|
792
|
+
return 0;
|
|
793
|
+
|
|
794
|
+
if (!setup_easy_handle(ctx, rb_ary_entry(requests, idx), timeout_sec))
|
|
795
|
+
return 0;
|
|
796
|
+
|
|
797
|
+
CURLMcode mc = curl_multi_add_handle(session->multi, ctx->easy);
|
|
798
|
+
if (mc != CURLM_OK)
|
|
799
|
+
return 0;
|
|
800
|
+
|
|
801
|
+
ctx->active = 1;
|
|
802
|
+
ctx->done = 0;
|
|
803
|
+
session->active_count++;
|
|
804
|
+
return 1;
|
|
805
|
+
}
|
|
806
|
+
|
|
807
|
+
static int fill_slots(multi_session_t *session, VALUE requests, int *invalid, long timeout_sec,
|
|
808
|
+
completion_ctx_t *cctx) {
|
|
809
|
+
while (session->active_count < session->max_connections) {
|
|
810
|
+
int idx = next_pending_index(session);
|
|
811
|
+
if (idx < 0)
|
|
812
|
+
break;
|
|
813
|
+
|
|
814
|
+
request_ctx_t *ctx = &session->requests[idx];
|
|
815
|
+
|
|
816
|
+
if (!activate_request(session, requests, idx, invalid, timeout_sec)) {
|
|
817
|
+
invalid[idx] = 1;
|
|
818
|
+
ctx->done = 1;
|
|
819
|
+
ctx->active = 0;
|
|
820
|
+
if (record_immediate_error(cctx, idx, "Invalid request configuration"))
|
|
821
|
+
return 1;
|
|
596
822
|
}
|
|
597
823
|
}
|
|
824
|
+
|
|
825
|
+
return 0;
|
|
826
|
+
}
|
|
827
|
+
|
|
828
|
+
static int pending_remaining(multi_session_t *session) {
|
|
829
|
+
return session->pending_pos < session->pending_count;
|
|
830
|
+
}
|
|
831
|
+
|
|
832
|
+
static void prepare_pending(multi_session_t *session, int *indices, int count) {
|
|
833
|
+
session->pending_indices = indices;
|
|
834
|
+
session->pending_count = count;
|
|
835
|
+
session->pending_pos = 0;
|
|
836
|
+
session->active_count = 0;
|
|
837
|
+
session->still_running = 0;
|
|
838
|
+
}
|
|
839
|
+
|
|
840
|
+
static void run_multi_loop(multi_session_t *session, completion_ctx_t *cctx, VALUE requests,
|
|
841
|
+
int *invalid, long timeout_sec, int *indices, int indices_count) {
|
|
842
|
+
#ifdef FAST_CURL_HAVE_FIBER_SCHEDULER
|
|
843
|
+
VALUE scheduler = current_fiber_scheduler();
|
|
844
|
+
#endif
|
|
845
|
+
prepare_pending(session, indices, indices_count);
|
|
846
|
+
|
|
847
|
+
if (fill_slots(session, requests, invalid, timeout_sec, cctx))
|
|
848
|
+
return;
|
|
849
|
+
|
|
850
|
+
curl_multi_perform(session->multi, &session->still_running);
|
|
851
|
+
if (process_completed(session, cctx))
|
|
852
|
+
return;
|
|
853
|
+
|
|
854
|
+
while (!session->cancelled && (session->active_count > 0 || pending_remaining(session))) {
|
|
855
|
+
if (fill_slots(session, requests, invalid, timeout_sec, cctx))
|
|
856
|
+
return;
|
|
857
|
+
|
|
858
|
+
if (session->active_count == 0)
|
|
859
|
+
break;
|
|
860
|
+
|
|
861
|
+
#ifdef FAST_CURL_HAVE_FIBER_SCHEDULER
|
|
862
|
+
if (scheduler != Qnil)
|
|
863
|
+
run_via_fiber_worker(scheduler, poll_without_gvl, session);
|
|
864
|
+
else
|
|
865
|
+
#endif
|
|
866
|
+
rb_thread_call_without_gvl(poll_without_gvl, session, unblock_perform, session);
|
|
867
|
+
|
|
868
|
+
if (process_completed(session, cctx))
|
|
869
|
+
return;
|
|
870
|
+
}
|
|
871
|
+
|
|
872
|
+
process_completed(session, cctx);
|
|
598
873
|
}
|
|
599
874
|
|
|
600
875
|
static int is_default_retryable_curle(CURLcode code) {
|
|
601
|
-
for (int i = 0; i < DEFAULT_RETRYABLE_CURLE_COUNT; i++)
|
|
876
|
+
for (int i = 0; i < DEFAULT_RETRYABLE_CURLE_COUNT; i++)
|
|
602
877
|
if (DEFAULT_RETRYABLE_CURLE[i] == code)
|
|
603
878
|
return 1;
|
|
604
|
-
}
|
|
605
879
|
return 0;
|
|
606
880
|
}
|
|
607
881
|
|
|
608
|
-
static int should_retry(request_ctx_t *ctx, retry_config_t *
|
|
609
|
-
if (ctx->curl_result != CURLE_OK)
|
|
882
|
+
static int should_retry(request_ctx_t *ctx, retry_config_t *rc) {
|
|
883
|
+
if (ctx->curl_result != CURLE_OK)
|
|
610
884
|
return is_default_retryable_curle(ctx->curl_result);
|
|
611
|
-
}
|
|
612
885
|
|
|
613
|
-
|
|
614
|
-
|
|
615
|
-
|
|
616
|
-
return 1;
|
|
617
|
-
}
|
|
618
|
-
}
|
|
886
|
+
for (int i = 0; i < rc->retry_http_count; i++)
|
|
887
|
+
if (rc->retry_http_codes[i] == (int)ctx->http_status)
|
|
888
|
+
return 1;
|
|
619
889
|
|
|
620
890
|
return 0;
|
|
621
891
|
}
|
|
@@ -626,9 +896,8 @@ typedef struct {
|
|
|
626
896
|
|
|
627
897
|
static void *sleep_without_gvl(void *arg) {
|
|
628
898
|
sleep_arg_t *sa = (sleep_arg_t *)arg;
|
|
629
|
-
struct timespec ts
|
|
630
|
-
|
|
631
|
-
ts.tv_nsec = (sa->delay_ms % 1000) * 1000000L;
|
|
899
|
+
struct timespec ts = {.tv_sec = sa->delay_ms / 1000,
|
|
900
|
+
.tv_nsec = (sa->delay_ms % 1000) * 1000000L};
|
|
632
901
|
nanosleep(&ts, NULL);
|
|
633
902
|
return NULL;
|
|
634
903
|
}
|
|
@@ -637,193 +906,216 @@ static void retry_delay_sleep(long delay_ms) {
|
|
|
637
906
|
if (delay_ms <= 0)
|
|
638
907
|
return;
|
|
639
908
|
|
|
640
|
-
|
|
909
|
+
#ifdef FAST_CURL_HAVE_FIBER_SCHEDULER
|
|
910
|
+
VALUE scheduler = current_fiber_scheduler();
|
|
911
|
+
if (scheduler != Qnil) {
|
|
641
912
|
long remaining = delay_ms;
|
|
642
913
|
while (remaining > 0) {
|
|
643
914
|
long chunk = remaining > FIBER_POLL_TIMEOUT_MS ? FIBER_POLL_TIMEOUT_MS : remaining;
|
|
644
915
|
sleep_arg_t sa = {.delay_ms = chunk};
|
|
645
|
-
sleep_without_gvl
|
|
646
|
-
rb_thread_schedule();
|
|
916
|
+
run_via_fiber_worker(scheduler, sleep_without_gvl, &sa);
|
|
647
917
|
remaining -= chunk;
|
|
648
918
|
}
|
|
649
|
-
} else
|
|
919
|
+
} else
|
|
920
|
+
#endif
|
|
921
|
+
{
|
|
650
922
|
sleep_arg_t sa = {.delay_ms = delay_ms};
|
|
651
|
-
rb_thread_call_without_gvl(sleep_without_gvl, &sa,
|
|
923
|
+
rb_thread_call_without_gvl(sleep_without_gvl, &sa, (rb_unblock_function_t *)0, NULL);
|
|
652
924
|
}
|
|
653
925
|
}
|
|
654
926
|
|
|
655
|
-
static void
|
|
656
|
-
*timeout = 30;
|
|
657
|
-
*max_conn = 20;
|
|
927
|
+
static void retry_config_init(retry_config_t *retry_cfg) {
|
|
658
928
|
retry_cfg->max_retries = DEFAULT_RETRIES;
|
|
929
|
+
retry_cfg->retries_explicit = 0;
|
|
659
930
|
retry_cfg->retry_delay_ms = DEFAULT_RETRY_DELAY;
|
|
660
931
|
retry_cfg->retry_http_codes = NULL;
|
|
661
932
|
retry_cfg->retry_http_count = 0;
|
|
933
|
+
}
|
|
934
|
+
|
|
935
|
+
static long parse_long_option(VALUE options, key_id_t key, const char *name, long min, long max,
|
|
936
|
+
long default_value, int *present) {
|
|
937
|
+
VALUE raw = hash_aref_key(options, key);
|
|
938
|
+
long value;
|
|
939
|
+
|
|
940
|
+
if (present)
|
|
941
|
+
*present = !NIL_P(raw);
|
|
942
|
+
if (NIL_P(raw))
|
|
943
|
+
return default_value;
|
|
662
944
|
|
|
663
|
-
|
|
945
|
+
value = NUM2LONG(raw);
|
|
946
|
+
if (value < min || value > max)
|
|
947
|
+
rb_raise(rb_eArgError, "%s must be between %ld and %ld", name, min, max);
|
|
948
|
+
|
|
949
|
+
return value;
|
|
950
|
+
}
|
|
951
|
+
|
|
952
|
+
static int parse_int_option(VALUE options, key_id_t key, const char *name, int min, int max,
|
|
953
|
+
int default_value, int *present) {
|
|
954
|
+
return (int)parse_long_option(options, key, name, min, max, default_value, present);
|
|
955
|
+
}
|
|
956
|
+
|
|
957
|
+
static void parse_retry_codes(VALUE options, retry_config_t *retry_cfg) {
|
|
958
|
+
VALUE codes = hash_aref_key(options, KEY_RETRY_CODES);
|
|
959
|
+
long len_long;
|
|
960
|
+
int len;
|
|
961
|
+
|
|
962
|
+
if (NIL_P(codes))
|
|
664
963
|
return;
|
|
665
964
|
|
|
666
|
-
|
|
667
|
-
|
|
668
|
-
|
|
669
|
-
|
|
670
|
-
|
|
671
|
-
|
|
672
|
-
|
|
673
|
-
|
|
674
|
-
|
|
675
|
-
|
|
676
|
-
VALUE c = rb_hash_aref(options, sym_connections);
|
|
677
|
-
if (!NIL_P(c)) {
|
|
678
|
-
int conn_val = NUM2INT(c);
|
|
679
|
-
if (conn_val > 100)
|
|
680
|
-
conn_val = 100;
|
|
681
|
-
else if (conn_val <= 0)
|
|
682
|
-
conn_val = 20;
|
|
683
|
-
*max_conn = conn_val;
|
|
684
|
-
}
|
|
685
|
-
|
|
686
|
-
VALUE r = rb_hash_aref(options, sym_retries);
|
|
687
|
-
if (!NIL_P(r)) {
|
|
688
|
-
int retries_val = NUM2INT(r);
|
|
689
|
-
if (retries_val < 0)
|
|
690
|
-
retries_val = 0;
|
|
691
|
-
if (retries_val > MAX_RETRIES)
|
|
692
|
-
retries_val = MAX_RETRIES;
|
|
693
|
-
retry_cfg->max_retries = retries_val;
|
|
694
|
-
}
|
|
695
|
-
|
|
696
|
-
VALUE rd = rb_hash_aref(options, sym_retry_delay);
|
|
697
|
-
if (!NIL_P(rd)) {
|
|
698
|
-
long delay_val = NUM2LONG(rd);
|
|
699
|
-
if (delay_val < 0)
|
|
700
|
-
delay_val = 0;
|
|
701
|
-
if (delay_val > 30000)
|
|
702
|
-
delay_val = 30000;
|
|
703
|
-
retry_cfg->retry_delay_ms = delay_val;
|
|
704
|
-
}
|
|
705
|
-
|
|
706
|
-
VALUE rc = rb_hash_aref(options, sym_retry_codes);
|
|
707
|
-
if (!NIL_P(rc) && rb_obj_is_kind_of(rc, rb_cArray)) {
|
|
708
|
-
int len = (int)RARRAY_LEN(rc);
|
|
709
|
-
if (len > 0) {
|
|
710
|
-
retry_cfg->retry_http_codes = malloc(sizeof(int) * len);
|
|
711
|
-
if (retry_cfg->retry_http_codes) {
|
|
712
|
-
retry_cfg->retry_http_count = len;
|
|
713
|
-
for (int i = 0; i < len; i++) {
|
|
714
|
-
retry_cfg->retry_http_codes[i] = NUM2INT(rb_ary_entry(rc, i));
|
|
715
|
-
}
|
|
716
|
-
}
|
|
717
|
-
}
|
|
965
|
+
Check_Type(codes, T_ARRAY);
|
|
966
|
+
len_long = RARRAY_LEN(codes);
|
|
967
|
+
if (len_long > INT_MAX)
|
|
968
|
+
rb_raise(rb_eArgError, "retry_codes is too large");
|
|
969
|
+
|
|
970
|
+
len = (int)len_long;
|
|
971
|
+
for (int i = 0; i < len; i++) {
|
|
972
|
+
int code = NUM2INT(rb_ary_entry(codes, i));
|
|
973
|
+
if (code < 100 || code > 599)
|
|
974
|
+
rb_raise(rb_eArgError, "retry_codes must contain valid HTTP status codes");
|
|
718
975
|
}
|
|
976
|
+
|
|
977
|
+
if (len == 0)
|
|
978
|
+
return;
|
|
979
|
+
|
|
980
|
+
retry_cfg->retry_http_codes = malloc(sizeof(int) * (size_t)len);
|
|
981
|
+
if (!retry_cfg->retry_http_codes)
|
|
982
|
+
rb_raise(rb_eNoMemError, "failed to allocate retry codes");
|
|
983
|
+
|
|
984
|
+
retry_cfg->retry_http_count = len;
|
|
985
|
+
for (int i = 0; i < len; i++)
|
|
986
|
+
retry_cfg->retry_http_codes[i] = NUM2INT(rb_ary_entry(codes, i));
|
|
719
987
|
}
|
|
720
988
|
|
|
721
|
-
static
|
|
722
|
-
|
|
723
|
-
|
|
724
|
-
|
|
725
|
-
return rb_ary_new();
|
|
989
|
+
static void parse_options(VALUE options, long *timeout, int *max_conn, retry_config_t *retry_cfg) {
|
|
990
|
+
*timeout = 30;
|
|
991
|
+
*max_conn = 20;
|
|
992
|
+
retry_config_init(retry_cfg);
|
|
726
993
|
|
|
727
|
-
|
|
728
|
-
|
|
729
|
-
retry_config_t retry_cfg;
|
|
730
|
-
parse_options(options, &timeout_sec, &max_conn, &retry_cfg);
|
|
994
|
+
if (NIL_P(options))
|
|
995
|
+
return;
|
|
731
996
|
|
|
732
|
-
|
|
733
|
-
|
|
734
|
-
|
|
735
|
-
|
|
736
|
-
|
|
737
|
-
|
|
738
|
-
|
|
739
|
-
|
|
740
|
-
|
|
741
|
-
|
|
997
|
+
Check_Type(options, T_HASH);
|
|
998
|
+
*timeout = parse_long_option(options, KEY_TIMEOUT, "timeout", 1, MAX_TIMEOUT, *timeout, NULL);
|
|
999
|
+
*max_conn = parse_int_option(options, KEY_CONNECTIONS, "connections", 1, MAX_CONNECTIONS,
|
|
1000
|
+
*max_conn, NULL);
|
|
1001
|
+
retry_cfg->max_retries = parse_int_option(options, KEY_RETRIES, "retries", 0, MAX_RETRIES,
|
|
1002
|
+
retry_cfg->max_retries, &retry_cfg->retries_explicit);
|
|
1003
|
+
retry_cfg->retry_delay_ms =
|
|
1004
|
+
parse_long_option(options, KEY_RETRY_DELAY, "retry_delay", 0, MAX_RETRY_DELAY_MS,
|
|
1005
|
+
retry_cfg->retry_delay_ms, NULL);
|
|
1006
|
+
parse_retry_codes(options, retry_cfg);
|
|
1007
|
+
}
|
|
742
1008
|
|
|
743
|
-
|
|
744
|
-
|
|
745
|
-
session
|
|
746
|
-
session
|
|
747
|
-
session
|
|
1009
|
+
static void multi_session_init(multi_session_t *session, CURLM *multi, int count, int max_conn,
|
|
1010
|
+
long timeout_sec) {
|
|
1011
|
+
memset(session, 0, sizeof(*session));
|
|
1012
|
+
session->multi = multi;
|
|
1013
|
+
session->count = count;
|
|
1014
|
+
session->timeout_ms = timeout_sec * 1000;
|
|
1015
|
+
session->max_connections = max_conn;
|
|
1016
|
+
}
|
|
748
1017
|
|
|
749
|
-
|
|
750
|
-
curl_multi_setopt(
|
|
1018
|
+
static void multi_session_configure(CURLM *multi, int max_conn) {
|
|
1019
|
+
curl_multi_setopt(multi, CURLMOPT_MAXCONNECTS, (long)max_conn);
|
|
1020
|
+
curl_multi_setopt(multi, CURLMOPT_MAX_TOTAL_CONNECTIONS, (long)max_conn);
|
|
751
1021
|
#ifdef CURLPIPE_MULTIPLEX
|
|
752
|
-
curl_multi_setopt(
|
|
1022
|
+
curl_multi_setopt(multi, CURLMOPT_PIPELINING, CURLPIPE_MULTIPLEX);
|
|
753
1023
|
#endif
|
|
1024
|
+
}
|
|
754
1025
|
|
|
755
|
-
|
|
756
|
-
|
|
757
|
-
|
|
758
|
-
|
|
759
|
-
|
|
760
|
-
|
|
1026
|
+
typedef struct {
|
|
1027
|
+
multi_session_t *session;
|
|
1028
|
+
int *invalid;
|
|
1029
|
+
retry_config_t *retry_cfg;
|
|
1030
|
+
} cleanup_ctx_t;
|
|
1031
|
+
|
|
1032
|
+
static VALUE cleanup_session(VALUE arg) {
|
|
1033
|
+
cleanup_ctx_t *ctx = (cleanup_ctx_t *)arg;
|
|
1034
|
+
|
|
1035
|
+
if (ctx->session->requests) {
|
|
1036
|
+
for (int i = 0; i < ctx->session->count; i++) {
|
|
1037
|
+
if (ctx->session->requests[i].easy && ctx->session->requests[i].active)
|
|
1038
|
+
curl_multi_remove_handle(ctx->session->multi, ctx->session->requests[i].easy);
|
|
1039
|
+
request_ctx_free(&ctx->session->requests[i]);
|
|
1040
|
+
}
|
|
1041
|
+
free(ctx->session->requests);
|
|
1042
|
+
ctx->session->requests = NULL;
|
|
761
1043
|
}
|
|
762
1044
|
|
|
763
|
-
|
|
764
|
-
|
|
765
|
-
|
|
766
|
-
curl_multi_cleanup(session.multi);
|
|
767
|
-
if (retry_cfg.retry_http_codes)
|
|
768
|
-
free(retry_cfg.retry_http_codes);
|
|
769
|
-
rb_raise(rb_eNoMemError, "failed to allocate tracking array");
|
|
1045
|
+
if (ctx->invalid) {
|
|
1046
|
+
free(ctx->invalid);
|
|
1047
|
+
ctx->invalid = NULL;
|
|
770
1048
|
}
|
|
771
1049
|
|
|
772
|
-
|
|
773
|
-
|
|
774
|
-
|
|
775
|
-
|
|
1050
|
+
if (ctx->session->multi) {
|
|
1051
|
+
curl_multi_cleanup(ctx->session->multi);
|
|
1052
|
+
ctx->session->multi = NULL;
|
|
1053
|
+
}
|
|
776
1054
|
|
|
777
|
-
|
|
778
|
-
|
|
779
|
-
|
|
780
|
-
|
|
781
|
-
}
|
|
1055
|
+
if (ctx->retry_cfg && ctx->retry_cfg->retry_http_codes) {
|
|
1056
|
+
free(ctx->retry_cfg->retry_http_codes);
|
|
1057
|
+
ctx->retry_cfg->retry_http_codes = NULL;
|
|
1058
|
+
}
|
|
782
1059
|
|
|
783
|
-
|
|
784
|
-
|
|
785
|
-
session.requests[i].done = 1;
|
|
786
|
-
invalid[i] = 1;
|
|
787
|
-
continue;
|
|
788
|
-
}
|
|
1060
|
+
return Qnil;
|
|
1061
|
+
}
|
|
789
1062
|
|
|
790
|
-
|
|
791
|
-
|
|
1063
|
+
static FAST_CURL_NORETURN void cleanup_and_raise(cleanup_ctx_t *cleanup, VALUE exception,
|
|
1064
|
+
const char *message) {
|
|
1065
|
+
cleanup_session((VALUE)cleanup);
|
|
1066
|
+
rb_raise(exception, "%s", message);
|
|
1067
|
+
}
|
|
792
1068
|
|
|
793
|
-
|
|
794
|
-
|
|
1069
|
+
typedef struct {
|
|
1070
|
+
VALUE requests;
|
|
1071
|
+
VALUE options;
|
|
1072
|
+
int target;
|
|
1073
|
+
int stream;
|
|
1074
|
+
multi_session_t *session;
|
|
1075
|
+
int *invalid;
|
|
1076
|
+
retry_config_t *retry_cfg;
|
|
1077
|
+
long timeout_sec;
|
|
1078
|
+
} execute_args_t;
|
|
1079
|
+
|
|
1080
|
+
static VALUE internal_execute_body(VALUE arg) {
|
|
1081
|
+
execute_args_t *ea = (execute_args_t *)arg;
|
|
1082
|
+
VALUE requests = ea->requests;
|
|
1083
|
+
multi_session_t *session = ea->session;
|
|
1084
|
+
int *invalid = ea->invalid;
|
|
1085
|
+
retry_config_t *retry_cfg = ea->retry_cfg;
|
|
1086
|
+
long timeout_sec = ea->timeout_sec;
|
|
1087
|
+
int count = session->count;
|
|
1088
|
+
int target = ea->target;
|
|
1089
|
+
int stream = ea->stream;
|
|
1090
|
+
|
|
1091
|
+
for (int i = 0; i < count; i++)
|
|
1092
|
+
request_ctx_init(&session->requests[i], i);
|
|
795
1093
|
|
|
796
1094
|
completion_ctx_t cctx;
|
|
797
|
-
cctx.results = stream ? Qnil : rb_ary_new2(count);
|
|
1095
|
+
cctx.results = stream ? Qnil : ((target > 0) ? rb_ary_new2(target) : rb_ary_new2(count));
|
|
798
1096
|
cctx.completed = 0;
|
|
799
1097
|
cctx.target = target;
|
|
800
1098
|
cctx.stream = stream;
|
|
801
1099
|
|
|
802
|
-
if (!stream) {
|
|
1100
|
+
if (!stream && target <= 0) {
|
|
803
1101
|
for (int i = 0; i < count; i++)
|
|
804
1102
|
rb_ary_store(cctx.results, i, Qnil);
|
|
805
1103
|
}
|
|
806
1104
|
|
|
807
|
-
run_multi_loop(
|
|
808
|
-
|
|
809
|
-
/* === Retry loop (execute mode only) === */
|
|
810
|
-
if (!stream && retry_cfg.max_retries > 0) {
|
|
811
|
-
int prev_all_failed = 0;
|
|
1105
|
+
run_multi_loop(session, &cctx, requests, invalid, timeout_sec, NULL, count);
|
|
812
1106
|
|
|
813
|
-
|
|
814
|
-
|
|
815
|
-
int *retry_indices = malloc(sizeof(int) * count);
|
|
1107
|
+
if (!stream && target <= 0 && retry_cfg->max_retries > 0) {
|
|
1108
|
+
for (int attempt = 0; attempt < retry_cfg->max_retries; attempt++) {
|
|
1109
|
+
int *retry_indices = malloc(sizeof(int) * (size_t)count);
|
|
816
1110
|
if (!retry_indices)
|
|
817
|
-
|
|
1111
|
+
rb_raise(rb_eNoMemError, "failed to allocate retry index array");
|
|
818
1112
|
|
|
1113
|
+
int retry_count = 0;
|
|
819
1114
|
for (int i = 0; i < count; i++) {
|
|
820
|
-
if (invalid[i])
|
|
1115
|
+
if (invalid[i] || !session->requests[i].done)
|
|
821
1116
|
continue;
|
|
822
|
-
if (
|
|
823
|
-
continue;
|
|
824
|
-
if (should_retry(&session.requests[i], &retry_cfg)) {
|
|
1117
|
+
if (should_retry(&session->requests[i], retry_cfg))
|
|
825
1118
|
retry_indices[retry_count++] = i;
|
|
826
|
-
}
|
|
827
1119
|
}
|
|
828
1120
|
|
|
829
1121
|
if (retry_count == 0) {
|
|
@@ -831,90 +1123,211 @@ static VALUE internal_execute(VALUE requests, VALUE options, int target, int str
|
|
|
831
1123
|
break;
|
|
832
1124
|
}
|
|
833
1125
|
|
|
834
|
-
|
|
835
|
-
for (int i = 0; i < count; i++) {
|
|
836
|
-
if (!invalid[i] && session.requests[i].done)
|
|
837
|
-
done_count++;
|
|
838
|
-
}
|
|
839
|
-
|
|
840
|
-
int all_failed_this_round = (retry_count == done_count);
|
|
841
|
-
|
|
842
|
-
if (all_failed_this_round && prev_all_failed) {
|
|
843
|
-
free(retry_indices);
|
|
844
|
-
break;
|
|
845
|
-
}
|
|
846
|
-
|
|
847
|
-
prev_all_failed = all_failed_this_round;
|
|
848
|
-
|
|
849
|
-
retry_delay_sleep(retry_cfg.retry_delay_ms);
|
|
1126
|
+
retry_delay_sleep(retry_cfg->retry_delay_ms);
|
|
850
1127
|
|
|
1128
|
+
int runnable_count = 0;
|
|
851
1129
|
for (int r = 0; r < retry_count; r++) {
|
|
852
1130
|
int idx = retry_indices[r];
|
|
853
|
-
request_ctx_t *
|
|
1131
|
+
request_ctx_t *rc = &session->requests[idx];
|
|
854
1132
|
|
|
855
|
-
|
|
856
|
-
|
|
857
|
-
if (!request_ctx_reset_for_retry(ctx)) {
|
|
858
|
-
ctx->done = 1;
|
|
1133
|
+
if (!request_ctx_reset_for_retry(rc)) {
|
|
859
1134
|
invalid[idx] = 1;
|
|
1135
|
+
rc->done = 1;
|
|
860
1136
|
continue;
|
|
861
1137
|
}
|
|
862
1138
|
|
|
863
|
-
|
|
864
|
-
|
|
865
|
-
ctx->done = 1;
|
|
866
|
-
invalid[idx] = 1;
|
|
867
|
-
continue;
|
|
868
|
-
}
|
|
1139
|
+
retry_indices[runnable_count++] = idx;
|
|
1140
|
+
}
|
|
869
1141
|
|
|
870
|
-
|
|
871
|
-
|
|
872
|
-
|
|
873
|
-
|
|
874
|
-
}
|
|
1142
|
+
if (runnable_count > 0) {
|
|
1143
|
+
cctx.completed = 0;
|
|
1144
|
+
run_multi_loop(session, &cctx, requests, invalid, timeout_sec, retry_indices,
|
|
1145
|
+
runnable_count);
|
|
875
1146
|
}
|
|
876
1147
|
|
|
877
1148
|
free(retry_indices);
|
|
878
|
-
|
|
879
|
-
cctx.completed = 0;
|
|
880
|
-
run_multi_loop(&session, &cctx);
|
|
881
1149
|
}
|
|
882
1150
|
}
|
|
883
1151
|
|
|
884
|
-
if (!stream) {
|
|
1152
|
+
if (!stream && target <= 0) {
|
|
885
1153
|
for (int i = 0; i < count; i++) {
|
|
886
|
-
request_ctx_t *
|
|
1154
|
+
request_ctx_t *rc = &session->requests[i];
|
|
1155
|
+
VALUE response;
|
|
887
1156
|
|
|
888
1157
|
if (invalid[i]) {
|
|
889
|
-
|
|
890
|
-
|
|
891
|
-
|
|
892
|
-
|
|
893
|
-
|
|
894
|
-
|
|
895
|
-
VALUE response;
|
|
896
|
-
if (ctx->curl_result == CURLE_OK) {
|
|
897
|
-
response = build_response(ctx);
|
|
1158
|
+
response = build_error_response("Invalid request configuration");
|
|
1159
|
+
} else if (!rc->done) {
|
|
1160
|
+
response = build_error_response("Request was not completed");
|
|
1161
|
+
} else if (rc->curl_result == CURLE_OK) {
|
|
1162
|
+
response = build_response(rc);
|
|
898
1163
|
} else {
|
|
899
|
-
response = build_error_response_with_code(curl_easy_strerror(
|
|
900
|
-
(int)
|
|
1164
|
+
response = build_error_response_with_code(curl_easy_strerror(rc->curl_result),
|
|
1165
|
+
(int)rc->curl_result);
|
|
901
1166
|
}
|
|
902
|
-
|
|
903
|
-
rb_ary_store(cctx.results, i,
|
|
1167
|
+
|
|
1168
|
+
rb_ary_store(cctx.results, i, build_result_pair(i, response));
|
|
904
1169
|
}
|
|
905
1170
|
}
|
|
906
1171
|
|
|
907
|
-
|
|
908
|
-
|
|
909
|
-
|
|
1172
|
+
return stream ? Qnil : cctx.results;
|
|
1173
|
+
}
|
|
1174
|
+
|
|
1175
|
+
#ifdef FAST_CURL_HAVE_FIBER_SCHEDULER
|
|
1176
|
+
typedef struct {
|
|
1177
|
+
execute_args_t *ea;
|
|
1178
|
+
VALUE scheduler;
|
|
1179
|
+
VALUE blocker;
|
|
1180
|
+
VALUE fiber;
|
|
1181
|
+
VALUE thread;
|
|
1182
|
+
VALUE result;
|
|
1183
|
+
VALUE exception;
|
|
1184
|
+
int state;
|
|
1185
|
+
int finished;
|
|
1186
|
+
} scheduler_execute_ctx_t;
|
|
1187
|
+
|
|
1188
|
+
static VALUE scheduler_execute_thread(void *arg) {
|
|
1189
|
+
scheduler_execute_ctx_t *ctx = (scheduler_execute_ctx_t *)arg;
|
|
1190
|
+
|
|
1191
|
+
ctx->result = rb_protect(internal_execute_body, (VALUE)ctx->ea, &ctx->state);
|
|
1192
|
+
if (ctx->state) {
|
|
1193
|
+
ctx->exception = rb_errinfo();
|
|
1194
|
+
rb_set_errinfo(Qnil);
|
|
910
1195
|
}
|
|
911
|
-
free(session.requests);
|
|
912
|
-
free(invalid);
|
|
913
|
-
curl_multi_cleanup(session.multi);
|
|
914
|
-
if (retry_cfg.retry_http_codes)
|
|
915
|
-
free(retry_cfg.retry_http_codes);
|
|
916
1196
|
|
|
917
|
-
|
|
1197
|
+
ctx->finished = 1;
|
|
1198
|
+
rb_fiber_scheduler_unblock(ctx->scheduler, ctx->blocker, ctx->fiber);
|
|
1199
|
+
return Qnil;
|
|
1200
|
+
}
|
|
1201
|
+
|
|
1202
|
+
static void cancel_scheduler_execute(scheduler_execute_ctx_t *ctx) {
|
|
1203
|
+
multi_session_t *session = ctx->ea->session;
|
|
1204
|
+
|
|
1205
|
+
session->cancelled = 1;
|
|
1206
|
+
#ifdef HAVE_CURL_MULTI_WAKEUP
|
|
1207
|
+
if (session->multi)
|
|
1208
|
+
curl_multi_wakeup(session->multi);
|
|
1209
|
+
#endif
|
|
1210
|
+
}
|
|
1211
|
+
|
|
1212
|
+
static VALUE scheduler_execute_wait(VALUE arg) {
|
|
1213
|
+
scheduler_execute_ctx_t *ctx = (scheduler_execute_ctx_t *)arg;
|
|
1214
|
+
|
|
1215
|
+
if (!ctx->finished)
|
|
1216
|
+
rb_fiber_scheduler_block(ctx->scheduler, ctx->blocker, Qnil);
|
|
1217
|
+
|
|
1218
|
+
rb_funcall(ctx->thread, rb_intern("join"), 0);
|
|
1219
|
+
|
|
1220
|
+
if (ctx->state)
|
|
1221
|
+
rb_exc_raise(ctx->exception);
|
|
1222
|
+
|
|
1223
|
+
return ctx->result;
|
|
1224
|
+
}
|
|
1225
|
+
|
|
1226
|
+
static VALUE scheduler_execute_ensure(VALUE arg) {
|
|
1227
|
+
scheduler_execute_ctx_t *ctx = (scheduler_execute_ctx_t *)arg;
|
|
1228
|
+
|
|
1229
|
+
if (!NIL_P(ctx->thread)) {
|
|
1230
|
+
if (!ctx->finished)
|
|
1231
|
+
cancel_scheduler_execute(ctx);
|
|
1232
|
+
rb_funcall(ctx->thread, rb_intern("join"), 0);
|
|
1233
|
+
}
|
|
1234
|
+
|
|
1235
|
+
return Qnil;
|
|
1236
|
+
}
|
|
1237
|
+
|
|
1238
|
+
static VALUE execute_with_fiber_scheduler(VALUE arg) {
|
|
1239
|
+
scheduler_execute_ctx_t *ctx = (scheduler_execute_ctx_t *)arg;
|
|
1240
|
+
|
|
1241
|
+
ctx->blocker = rb_obj_alloc(rb_cObject);
|
|
1242
|
+
ctx->fiber = rb_fiber_current();
|
|
1243
|
+
ctx->thread = rb_thread_create(scheduler_execute_thread, ctx);
|
|
1244
|
+
|
|
1245
|
+
return rb_ensure(scheduler_execute_wait, arg, scheduler_execute_ensure, arg);
|
|
1246
|
+
}
|
|
1247
|
+
#endif
|
|
1248
|
+
|
|
1249
|
+
static VALUE internal_execute(VALUE requests, VALUE options, int target, int stream) {
|
|
1250
|
+
Check_Type(requests, T_ARRAY);
|
|
1251
|
+
|
|
1252
|
+
long count_long = RARRAY_LEN(requests);
|
|
1253
|
+
if (count_long == 0)
|
|
1254
|
+
return rb_ary_new();
|
|
1255
|
+
if (count_long > MAX_REQUESTS)
|
|
1256
|
+
rb_raise(rb_eArgError, "too many requests (%ld), maximum is %d", count_long, MAX_REQUESTS);
|
|
1257
|
+
if (count_long > INT_MAX)
|
|
1258
|
+
rb_raise(rb_eArgError, "request count overflows int");
|
|
1259
|
+
|
|
1260
|
+
int count = (int)count_long;
|
|
1261
|
+
|
|
1262
|
+
if (target > 0 && target > count)
|
|
1263
|
+
target = count;
|
|
1264
|
+
|
|
1265
|
+
long timeout_sec;
|
|
1266
|
+
int max_conn;
|
|
1267
|
+
retry_config_t retry_cfg;
|
|
1268
|
+
parse_options(options, &timeout_sec, &max_conn, &retry_cfg);
|
|
1269
|
+
|
|
1270
|
+
if (stream || target > 0) {
|
|
1271
|
+
if (retry_cfg.retries_explicit && retry_cfg.max_retries > 0 && stream)
|
|
1272
|
+
rb_warn(
|
|
1273
|
+
"FastCurl: retries are not supported in stream_execute, ignoring retries option");
|
|
1274
|
+
if (retry_cfg.retries_explicit && retry_cfg.max_retries > 0 && target > 0)
|
|
1275
|
+
rb_warn(
|
|
1276
|
+
"FastCurl: retries are not supported in first_execute, ignoring retries option");
|
|
1277
|
+
retry_cfg.max_retries = 0;
|
|
1278
|
+
}
|
|
1279
|
+
|
|
1280
|
+
multi_session_t session;
|
|
1281
|
+
int *invalid = NULL;
|
|
1282
|
+
multi_session_init(&session, curl_multi_init(), count, max_conn, timeout_sec);
|
|
1283
|
+
|
|
1284
|
+
cleanup_ctx_t cleanup = {.session = &session, .invalid = NULL, .retry_cfg = &retry_cfg};
|
|
1285
|
+
|
|
1286
|
+
if (!session.multi)
|
|
1287
|
+
cleanup_and_raise(&cleanup, rb_eNoMemError, "failed to initialize curl multi handle");
|
|
1288
|
+
|
|
1289
|
+
multi_session_configure(session.multi, max_conn);
|
|
1290
|
+
|
|
1291
|
+
session.requests = calloc((size_t)count, sizeof(request_ctx_t));
|
|
1292
|
+
if (!session.requests)
|
|
1293
|
+
cleanup_and_raise(&cleanup, rb_eNoMemError, "failed to allocate request contexts");
|
|
1294
|
+
|
|
1295
|
+
invalid = calloc((size_t)count, sizeof(int));
|
|
1296
|
+
cleanup.invalid = invalid;
|
|
1297
|
+
if (!invalid)
|
|
1298
|
+
cleanup_and_raise(&cleanup, rb_eNoMemError, "failed to allocate tracking array");
|
|
1299
|
+
execute_args_t ea = {
|
|
1300
|
+
.requests = requests,
|
|
1301
|
+
.options = options,
|
|
1302
|
+
.target = target,
|
|
1303
|
+
.stream = stream,
|
|
1304
|
+
.session = &session,
|
|
1305
|
+
.invalid = invalid,
|
|
1306
|
+
.retry_cfg = &retry_cfg,
|
|
1307
|
+
.timeout_sec = timeout_sec,
|
|
1308
|
+
};
|
|
1309
|
+
|
|
1310
|
+
#ifdef FAST_CURL_HAVE_FIBER_SCHEDULER
|
|
1311
|
+
VALUE scheduler = current_fiber_scheduler();
|
|
1312
|
+
if (scheduler != Qnil && !stream) {
|
|
1313
|
+
scheduler_execute_ctx_t scheduler_ctx = {
|
|
1314
|
+
.ea = &ea,
|
|
1315
|
+
.scheduler = scheduler,
|
|
1316
|
+
.blocker = Qnil,
|
|
1317
|
+
.fiber = Qnil,
|
|
1318
|
+
.thread = Qnil,
|
|
1319
|
+
.result = Qnil,
|
|
1320
|
+
.exception = Qnil,
|
|
1321
|
+
.state = 0,
|
|
1322
|
+
.finished = 0,
|
|
1323
|
+
};
|
|
1324
|
+
|
|
1325
|
+
return rb_ensure(execute_with_fiber_scheduler, (VALUE)&scheduler_ctx, cleanup_session,
|
|
1326
|
+
(VALUE)&cleanup);
|
|
1327
|
+
}
|
|
1328
|
+
#endif
|
|
1329
|
+
|
|
1330
|
+
return rb_ensure(internal_execute_body, (VALUE)&ea, cleanup_session, (VALUE)&cleanup);
|
|
918
1331
|
}
|
|
919
1332
|
|
|
920
1333
|
static VALUE rb_fast_curl_execute(int argc, VALUE *argv, VALUE self) {
|
|
@@ -929,11 +1342,15 @@ static VALUE rb_fast_curl_first_execute(int argc, VALUE *argv, VALUE self) {
|
|
|
929
1342
|
|
|
930
1343
|
int count = 1;
|
|
931
1344
|
if (!NIL_P(options)) {
|
|
932
|
-
|
|
1345
|
+
Check_Type(options, T_HASH);
|
|
1346
|
+
VALUE c = hash_aref_key(options, KEY_COUNT_OPT);
|
|
933
1347
|
if (!NIL_P(c))
|
|
934
1348
|
count = NUM2INT(c);
|
|
935
1349
|
}
|
|
936
1350
|
|
|
1351
|
+
if (count <= 0)
|
|
1352
|
+
rb_raise(rb_eArgError, "count must be positive");
|
|
1353
|
+
|
|
937
1354
|
return internal_execute(requests, options, count, 0);
|
|
938
1355
|
}
|
|
939
1356
|
|
|
@@ -950,44 +1367,11 @@ static VALUE rb_fast_curl_stream_execute(int argc, VALUE *argv, VALUE self) {
|
|
|
950
1367
|
void Init_fast_curl(void) {
|
|
951
1368
|
curl_global_init(CURL_GLOBAL_ALL);
|
|
952
1369
|
|
|
953
|
-
|
|
954
|
-
|
|
955
|
-
|
|
956
|
-
|
|
957
|
-
|
|
958
|
-
id_method = rb_intern("method");
|
|
959
|
-
id_timeout = rb_intern("timeout");
|
|
960
|
-
id_connections = rb_intern("connections");
|
|
961
|
-
id_count = rb_intern("count");
|
|
962
|
-
id_keys = rb_intern("keys");
|
|
963
|
-
id_retries = rb_intern("retries");
|
|
964
|
-
id_retry_delay = rb_intern("retry_delay");
|
|
965
|
-
id_retry_codes = rb_intern("retry_codes");
|
|
966
|
-
|
|
967
|
-
sym_status = ID2SYM(id_status);
|
|
968
|
-
rb_gc_register_address(&sym_status);
|
|
969
|
-
sym_headers = ID2SYM(id_headers);
|
|
970
|
-
rb_gc_register_address(&sym_headers);
|
|
971
|
-
sym_body = ID2SYM(id_body);
|
|
972
|
-
rb_gc_register_address(&sym_body);
|
|
973
|
-
sym_error_code = ID2SYM(id_error_code);
|
|
974
|
-
rb_gc_register_address(&sym_error_code);
|
|
975
|
-
sym_url = ID2SYM(id_url);
|
|
976
|
-
rb_gc_register_address(&sym_url);
|
|
977
|
-
sym_method = ID2SYM(id_method);
|
|
978
|
-
rb_gc_register_address(&sym_method);
|
|
979
|
-
sym_timeout = ID2SYM(id_timeout);
|
|
980
|
-
rb_gc_register_address(&sym_timeout);
|
|
981
|
-
sym_connections = ID2SYM(id_connections);
|
|
982
|
-
rb_gc_register_address(&sym_connections);
|
|
983
|
-
sym_count = ID2SYM(id_count);
|
|
984
|
-
rb_gc_register_address(&sym_count);
|
|
985
|
-
sym_retries = ID2SYM(id_retries);
|
|
986
|
-
rb_gc_register_address(&sym_retries);
|
|
987
|
-
sym_retry_delay = ID2SYM(id_retry_delay);
|
|
988
|
-
rb_gc_register_address(&sym_retry_delay);
|
|
989
|
-
sym_retry_codes = ID2SYM(id_retry_codes);
|
|
990
|
-
rb_gc_register_address(&sym_retry_codes);
|
|
1370
|
+
for (int i = 0; i < KEY_LAST; i++) {
|
|
1371
|
+
fast_ids[i] = rb_intern(KEY_NAMES[i]);
|
|
1372
|
+
fast_syms[i] = ID2SYM(fast_ids[i]);
|
|
1373
|
+
rb_gc_register_address(&fast_syms[i]);
|
|
1374
|
+
}
|
|
991
1375
|
|
|
992
1376
|
VALUE mFastCurl = rb_define_module("FastCurl");
|
|
993
1377
|
|