fast_curl 0.3.0 → 0.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +91 -11
- data/ext/fast_curl/extconf.rb +3 -0
- data/ext/fast_curl/fast_curl.c +1151 -359
- data/lib/fast_curl/version.rb +1 -1
- data/lib/fast_curl.rb +114 -6
- metadata +37 -27
data/ext/fast_curl/fast_curl.c
CHANGED
|
@@ -4,22 +4,46 @@
|
|
|
4
4
|
#ifdef HAVE_RUBY_FIBER_SCHEDULER_H
|
|
5
5
|
#include <ruby/fiber/scheduler.h>
|
|
6
6
|
#endif
|
|
7
|
+
|
|
8
|
+
#if defined(HAVE_RUBY_FIBER_SCHEDULER_H) && defined(HAVE_RB_FIBER_SCHEDULER_CURRENT) && \
|
|
9
|
+
defined(HAVE_RB_FIBER_SCHEDULER_BLOCK) && defined(HAVE_RB_FIBER_SCHEDULER_UNBLOCK) && \
|
|
10
|
+
defined(HAVE_RB_FIBER_CURRENT)
|
|
11
|
+
#define FAST_CURL_HAVE_FIBER_SCHEDULER 1
|
|
12
|
+
#endif
|
|
7
13
|
#include <curl/curl.h>
|
|
14
|
+
#include <ctype.h>
|
|
15
|
+
#include <limits.h>
|
|
16
|
+
#include <pthread.h>
|
|
17
|
+
#include <stdint.h>
|
|
8
18
|
#include <stdlib.h>
|
|
9
19
|
#include <string.h>
|
|
20
|
+
#include <strings.h>
|
|
21
|
+
#include <time.h>
|
|
22
|
+
|
|
23
|
+
#if defined(__GNUC__) || defined(__clang__)
|
|
24
|
+
#define FAST_CURL_NORETURN __attribute__((noreturn))
|
|
25
|
+
#else
|
|
26
|
+
#define FAST_CURL_NORETURN
|
|
27
|
+
#endif
|
|
10
28
|
|
|
11
|
-
#define MAX_RESPONSE_SIZE
|
|
12
|
-
#define MAX_REDIRECTS
|
|
13
|
-
#define MAX_TIMEOUT
|
|
14
|
-
#define MAX_RETRIES
|
|
15
|
-
#define MAX_REQUESTS
|
|
16
|
-
#define
|
|
17
|
-
#define
|
|
18
|
-
#define
|
|
19
|
-
#define
|
|
20
|
-
#define
|
|
21
|
-
#define
|
|
22
|
-
#define
|
|
29
|
+
#define MAX_RESPONSE_SIZE (100 * 1024 * 1024)
|
|
30
|
+
#define MAX_REDIRECTS 5
|
|
31
|
+
#define MAX_TIMEOUT 300
|
|
32
|
+
#define MAX_RETRIES 10
|
|
33
|
+
#define MAX_REQUESTS 10000
|
|
34
|
+
#define MAX_CONNECTIONS 100
|
|
35
|
+
#define MAX_RETRY_DELAY_MS 30000
|
|
36
|
+
#define DEFAULT_RETRIES 1
|
|
37
|
+
#define DEFAULT_RETRY_DELAY 100
|
|
38
|
+
#define DEFAULT_CONNECT_TIMEOUT_MS 10000L
|
|
39
|
+
#define MAX_CONNECT_TIMEOUT_MS 300000L
|
|
40
|
+
#define MAX_TOTAL_TIMEOUT_MS 3600000L
|
|
41
|
+
#define INITIAL_BUF_CAP 8192
|
|
42
|
+
#define INITIAL_HEADER_CAP 16
|
|
43
|
+
#define POLL_TIMEOUT_MS 50
|
|
44
|
+
#define POLL_SLICE_MS 2000
|
|
45
|
+
#define FIBER_POLL_TIMEOUT_MS 10
|
|
46
|
+
#define HEADER_LINE_BUF_SIZE 512
|
|
23
47
|
|
|
24
48
|
static const CURLcode DEFAULT_RETRYABLE_CURLE[] = {
|
|
25
49
|
CURLE_COULDNT_CONNECT, CURLE_OPERATION_TIMEDOUT, CURLE_SEND_ERROR, CURLE_RECV_ERROR,
|
|
@@ -27,12 +51,140 @@ static const CURLcode DEFAULT_RETRYABLE_CURLE[] = {
|
|
|
27
51
|
#define DEFAULT_RETRYABLE_CURLE_COUNT \
|
|
28
52
|
(int)(sizeof(DEFAULT_RETRYABLE_CURLE) / sizeof(DEFAULT_RETRYABLE_CURLE[0]))
|
|
29
53
|
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
54
|
+
typedef enum {
|
|
55
|
+
KEY_STATUS,
|
|
56
|
+
KEY_HEADERS,
|
|
57
|
+
KEY_BODY,
|
|
58
|
+
KEY_ERROR_CODE,
|
|
59
|
+
KEY_URL,
|
|
60
|
+
KEY_METHOD,
|
|
61
|
+
KEY_TIMEOUT,
|
|
62
|
+
KEY_CONNECTIONS,
|
|
63
|
+
KEY_COUNT_OPT,
|
|
64
|
+
KEY_RETRIES,
|
|
65
|
+
KEY_RETRY_DELAY,
|
|
66
|
+
KEY_RETRY_CODES,
|
|
67
|
+
KEY_CONNECT_TIMEOUT,
|
|
68
|
+
KEY_TOTAL_TIMEOUT,
|
|
69
|
+
KEY_RETRY_NON_IDEMPOTENT,
|
|
70
|
+
KEY_FOLLOW_REDIRECTS,
|
|
71
|
+
KEY_MAX_REDIRECTS,
|
|
72
|
+
KEY_EFFECTIVE_URL,
|
|
73
|
+
KEY_ERROR,
|
|
74
|
+
KEY_ATTEMPTS,
|
|
75
|
+
KEY_LAST
|
|
76
|
+
} key_id_t;
|
|
77
|
+
|
|
78
|
+
static ID fast_ids[KEY_LAST];
|
|
79
|
+
static VALUE fast_syms[KEY_LAST];
|
|
80
|
+
|
|
81
|
+
#define SYM(key) fast_syms[key]
|
|
82
|
+
|
|
83
|
+
static const char *const KEY_NAMES[KEY_LAST] = {
|
|
84
|
+
"status",
|
|
85
|
+
"headers",
|
|
86
|
+
"body",
|
|
87
|
+
"error_code",
|
|
88
|
+
"url",
|
|
89
|
+
"method",
|
|
90
|
+
"timeout",
|
|
91
|
+
"connections",
|
|
92
|
+
"count",
|
|
93
|
+
"retries",
|
|
94
|
+
"retry_delay",
|
|
95
|
+
"retry_codes",
|
|
96
|
+
"connect_timeout",
|
|
97
|
+
"total_timeout",
|
|
98
|
+
"retry_non_idempotent",
|
|
99
|
+
"follow_redirects",
|
|
100
|
+
"max_redirects",
|
|
101
|
+
"effective_url",
|
|
102
|
+
"error",
|
|
103
|
+
"attempts",
|
|
104
|
+
};
|
|
105
|
+
|
|
106
|
+
typedef enum {
|
|
107
|
+
ERR_CURL,
|
|
108
|
+
ERR_INVALID_REQUEST,
|
|
109
|
+
ERR_NOT_COMPLETED,
|
|
110
|
+
ERR_DEADLINE,
|
|
111
|
+
ERR_LAST
|
|
112
|
+
} error_kind_t;
|
|
113
|
+
|
|
114
|
+
static VALUE error_syms[ERR_LAST];
|
|
115
|
+
|
|
116
|
+
static const char *const ERROR_KIND_NAMES[ERR_LAST] = {
|
|
117
|
+
"curl_error",
|
|
118
|
+
"invalid_request",
|
|
119
|
+
"not_completed",
|
|
120
|
+
"deadline_exceeded",
|
|
121
|
+
};
|
|
122
|
+
|
|
123
|
+
static long long fast_now_ms(void) {
|
|
124
|
+
struct timespec ts;
|
|
125
|
+
#ifdef CLOCK_MONOTONIC
|
|
126
|
+
clock_gettime(CLOCK_MONOTONIC, &ts);
|
|
127
|
+
#else
|
|
128
|
+
clock_gettime(CLOCK_REALTIME, &ts);
|
|
129
|
+
#endif
|
|
130
|
+
return (long long)ts.tv_sec * 1000LL + ts.tv_nsec / 1000000LL;
|
|
131
|
+
}
|
|
132
|
+
|
|
133
|
+
static uint64_t fast_rand_state;
|
|
134
|
+
|
|
135
|
+
/* Small xorshift PRNG; only used to jitter retry delays. */
|
|
136
|
+
static uint32_t fast_rand(void) {
|
|
137
|
+
uint64_t x = fast_rand_state;
|
|
138
|
+
if (x == 0)
|
|
139
|
+
x = (uint64_t)fast_now_ms() | 1ULL;
|
|
140
|
+
x ^= x << 13;
|
|
141
|
+
x ^= x >> 7;
|
|
142
|
+
x ^= x << 17;
|
|
143
|
+
fast_rand_state = x;
|
|
144
|
+
return (uint32_t)(x >> 32);
|
|
145
|
+
}
|
|
146
|
+
|
|
147
|
+
/* Process-wide DNS and TLS session cache, so repeated calls to the same host
|
|
148
|
+
skip resolution and can resume TLS instead of doing a full handshake. */
|
|
149
|
+
static CURLSH *fast_share = NULL;
|
|
150
|
+
static pthread_mutex_t fast_share_locks[CURL_LOCK_DATA_LAST];
|
|
151
|
+
|
|
152
|
+
static void fast_share_lock(CURL *handle, curl_lock_data data, curl_lock_access access,
|
|
153
|
+
void *userptr) {
|
|
154
|
+
(void)handle;
|
|
155
|
+
(void)access;
|
|
156
|
+
(void)userptr;
|
|
157
|
+
if ((int)data > 0 && (int)data < CURL_LOCK_DATA_LAST)
|
|
158
|
+
pthread_mutex_lock(&fast_share_locks[data]);
|
|
159
|
+
}
|
|
160
|
+
|
|
161
|
+
static void fast_share_unlock(CURL *handle, curl_lock_data data, void *userptr) {
|
|
162
|
+
(void)handle;
|
|
163
|
+
(void)userptr;
|
|
164
|
+
if ((int)data > 0 && (int)data < CURL_LOCK_DATA_LAST)
|
|
165
|
+
pthread_mutex_unlock(&fast_share_locks[data]);
|
|
166
|
+
}
|
|
167
|
+
|
|
168
|
+
static void fast_share_init(void) {
|
|
169
|
+
for (int i = 0; i < CURL_LOCK_DATA_LAST; i++)
|
|
170
|
+
pthread_mutex_init(&fast_share_locks[i], NULL);
|
|
171
|
+
|
|
172
|
+
fast_share = curl_share_init();
|
|
173
|
+
if (!fast_share)
|
|
174
|
+
return;
|
|
175
|
+
|
|
176
|
+
curl_share_setopt(fast_share, CURLSHOPT_LOCKFUNC, fast_share_lock);
|
|
177
|
+
curl_share_setopt(fast_share, CURLSHOPT_UNLOCKFUNC, fast_share_unlock);
|
|
178
|
+
curl_share_setopt(fast_share, CURLSHOPT_SHARE, CURL_LOCK_DATA_DNS);
|
|
179
|
+
curl_share_setopt(fast_share, CURLSHOPT_SHARE, CURL_LOCK_DATA_SSL_SESSION);
|
|
180
|
+
|
|
181
|
+
/* CURL_LOCK_DATA_CONNECT is deliberately NOT shared. Sharing the connection
|
|
182
|
+
cache between multi handles running concurrently in different threads
|
|
183
|
+
deadlocks against CURLMOPT_MAX_TOTAL_CONNECTIONS, and removing that limit
|
|
184
|
+
turns the deadlock into a segfault inside curl_multi_perform (reproduced
|
|
185
|
+
on libcurl 8.5.0 with two threads x 25 requests). Cross-call TCP reuse
|
|
186
|
+
needs a persistent multi handle per thread, not a shared cache. */
|
|
187
|
+
}
|
|
36
188
|
|
|
37
189
|
typedef struct {
|
|
38
190
|
char *data;
|
|
@@ -64,20 +216,31 @@ static inline void buffer_reset(buffer_t *buf) {
|
|
|
64
216
|
static size_t write_callback(char *ptr, size_t size, size_t nmemb, void *userdata) {
|
|
65
217
|
buffer_t *buf = (buffer_t *)userdata;
|
|
66
218
|
size_t total = size * nmemb;
|
|
67
|
-
|
|
219
|
+
|
|
220
|
+
if (nmemb != 0 && total / nmemb != size)
|
|
221
|
+
return 0;
|
|
222
|
+
if (total > buf->max_size || buf->len > buf->max_size - total)
|
|
68
223
|
return 0;
|
|
224
|
+
|
|
69
225
|
if (buf->len + total >= buf->cap) {
|
|
70
226
|
size_t new_cap = (buf->cap == 0) ? INITIAL_BUF_CAP : buf->cap;
|
|
71
|
-
while (new_cap <= buf->len + total)
|
|
227
|
+
while (new_cap <= buf->len + total) {
|
|
228
|
+
if (new_cap > buf->max_size / 2) {
|
|
229
|
+
new_cap = buf->max_size;
|
|
230
|
+
break;
|
|
231
|
+
}
|
|
72
232
|
new_cap *= 2;
|
|
73
|
-
|
|
74
|
-
|
|
233
|
+
}
|
|
234
|
+
if (new_cap < buf->len + total)
|
|
235
|
+
return 0;
|
|
236
|
+
|
|
75
237
|
char *new_data = realloc(buf->data, new_cap);
|
|
76
238
|
if (!new_data)
|
|
77
239
|
return 0;
|
|
78
240
|
buf->data = new_data;
|
|
79
241
|
buf->cap = new_cap;
|
|
80
242
|
}
|
|
243
|
+
|
|
81
244
|
memcpy(buf->data + buf->len, ptr, total);
|
|
82
245
|
buf->len += total;
|
|
83
246
|
return total;
|
|
@@ -118,8 +281,22 @@ static void header_list_reset(header_list_t *h) {
|
|
|
118
281
|
static size_t header_callback(char *ptr, size_t size, size_t nmemb, void *userdata) {
|
|
119
282
|
header_list_t *h = (header_list_t *)userdata;
|
|
120
283
|
size_t total = size * nmemb;
|
|
121
|
-
|
|
284
|
+
|
|
285
|
+
if (nmemb != 0 && total / nmemb != size)
|
|
286
|
+
return 0;
|
|
287
|
+
|
|
288
|
+
size_t stripped = total;
|
|
289
|
+
while (stripped > 0 && (ptr[stripped - 1] == '\r' || ptr[stripped - 1] == '\n'))
|
|
290
|
+
stripped--;
|
|
291
|
+
|
|
292
|
+
if (stripped >= 5 && memcmp(ptr, "HTTP/", 5) == 0) {
|
|
293
|
+
header_list_reset(h);
|
|
294
|
+
return total;
|
|
295
|
+
}
|
|
296
|
+
|
|
297
|
+
if (stripped == 0)
|
|
122
298
|
return total;
|
|
299
|
+
|
|
123
300
|
if (h->count >= h->cap) {
|
|
124
301
|
int new_cap = (h->cap == 0) ? INITIAL_HEADER_CAP : h->cap * 2;
|
|
125
302
|
header_entry_t *ne = realloc(h->entries, sizeof(header_entry_t) * new_cap);
|
|
@@ -128,18 +305,17 @@ static size_t header_callback(char *ptr, size_t size, size_t nmemb, void *userda
|
|
|
128
305
|
h->entries = ne;
|
|
129
306
|
h->cap = new_cap;
|
|
130
307
|
}
|
|
131
|
-
|
|
132
|
-
while (stripped > 0 && (ptr[stripped - 1] == '\r' || ptr[stripped - 1] == '\n'))
|
|
133
|
-
stripped--;
|
|
308
|
+
|
|
134
309
|
char *entry = malloc(stripped + 1);
|
|
135
310
|
if (!entry)
|
|
136
311
|
return 0;
|
|
137
312
|
memcpy(entry, ptr, stripped);
|
|
138
313
|
entry[stripped] = '\0';
|
|
314
|
+
|
|
139
315
|
h->entries[h->count].str = entry;
|
|
140
316
|
h->entries[h->count].len = stripped;
|
|
141
317
|
h->count++;
|
|
142
|
-
return
|
|
318
|
+
return total;
|
|
143
319
|
}
|
|
144
320
|
|
|
145
321
|
typedef struct {
|
|
@@ -149,19 +325,36 @@ typedef struct {
|
|
|
149
325
|
header_list_t headers;
|
|
150
326
|
struct curl_slist *req_headers;
|
|
151
327
|
int done;
|
|
328
|
+
int active;
|
|
152
329
|
CURLcode curl_result;
|
|
153
330
|
long http_status;
|
|
331
|
+
int idempotent;
|
|
332
|
+
int attempts;
|
|
333
|
+
const char *setup_error;
|
|
334
|
+
int setup_error_fatal;
|
|
154
335
|
} request_ctx_t;
|
|
155
336
|
|
|
337
|
+
static VALUE fast_validation_error = Qnil;
|
|
338
|
+
#define SET_SETUP_ERROR(ctx, msg, fatal) \
|
|
339
|
+
do { \
|
|
340
|
+
(ctx)->setup_error = (msg); \
|
|
341
|
+
(ctx)->setup_error_fatal = (fatal); \
|
|
342
|
+
} while (0)
|
|
343
|
+
|
|
156
344
|
static inline void request_ctx_init(request_ctx_t *ctx, int index) {
|
|
157
|
-
ctx->easy =
|
|
345
|
+
ctx->easy = NULL;
|
|
158
346
|
ctx->index = index;
|
|
159
347
|
buffer_init(&ctx->body);
|
|
160
348
|
header_list_init(&ctx->headers);
|
|
161
349
|
ctx->req_headers = NULL;
|
|
162
350
|
ctx->done = 0;
|
|
351
|
+
ctx->active = 0;
|
|
163
352
|
ctx->curl_result = CURLE_OK;
|
|
164
353
|
ctx->http_status = 0;
|
|
354
|
+
ctx->idempotent = 1;
|
|
355
|
+
ctx->attempts = 0;
|
|
356
|
+
ctx->setup_error = NULL;
|
|
357
|
+
ctx->setup_error_fatal = 0;
|
|
165
358
|
}
|
|
166
359
|
|
|
167
360
|
static void request_ctx_free(request_ctx_t *ctx) {
|
|
@@ -175,6 +368,16 @@ static void request_ctx_free(request_ctx_t *ctx) {
|
|
|
175
368
|
curl_slist_free_all(ctx->req_headers);
|
|
176
369
|
ctx->req_headers = NULL;
|
|
177
370
|
}
|
|
371
|
+
ctx->active = 0;
|
|
372
|
+
}
|
|
373
|
+
|
|
374
|
+
static int request_ctx_prepare_easy(request_ctx_t *ctx) {
|
|
375
|
+
if (!ctx->easy) {
|
|
376
|
+
ctx->easy = curl_easy_init();
|
|
377
|
+
if (!ctx->easy)
|
|
378
|
+
return 0;
|
|
379
|
+
}
|
|
380
|
+
return 1;
|
|
178
381
|
}
|
|
179
382
|
|
|
180
383
|
static int request_ctx_reset_for_retry(request_ctx_t *ctx) {
|
|
@@ -192,8 +395,11 @@ static int request_ctx_reset_for_retry(request_ctx_t *ctx) {
|
|
|
192
395
|
if (!ctx->easy)
|
|
193
396
|
return 0;
|
|
194
397
|
ctx->done = 0;
|
|
398
|
+
ctx->active = 0;
|
|
195
399
|
ctx->curl_result = CURLE_OK;
|
|
196
400
|
ctx->http_status = 0;
|
|
401
|
+
ctx->setup_error = NULL;
|
|
402
|
+
ctx->setup_error_fatal = 0;
|
|
197
403
|
return 1;
|
|
198
404
|
}
|
|
199
405
|
|
|
@@ -205,15 +411,31 @@ typedef struct {
|
|
|
205
411
|
long timeout_ms;
|
|
206
412
|
int max_connections;
|
|
207
413
|
volatile int cancelled;
|
|
414
|
+
|
|
415
|
+
int active_count;
|
|
416
|
+
int pending_pos;
|
|
417
|
+
int pending_count;
|
|
418
|
+
int *pending_indices;
|
|
208
419
|
} multi_session_t;
|
|
209
420
|
|
|
210
421
|
typedef struct {
|
|
211
422
|
int max_retries;
|
|
423
|
+
int retries_explicit;
|
|
212
424
|
long retry_delay_ms;
|
|
213
425
|
int *retry_http_codes;
|
|
214
426
|
int retry_http_count;
|
|
427
|
+
int retry_non_idempotent;
|
|
215
428
|
} retry_config_t;
|
|
216
429
|
|
|
430
|
+
typedef struct {
|
|
431
|
+
long timeout_sec;
|
|
432
|
+
long connect_timeout_ms;
|
|
433
|
+
long total_timeout_ms;
|
|
434
|
+
long follow_redirects;
|
|
435
|
+
long max_redirects;
|
|
436
|
+
long long deadline_ms;
|
|
437
|
+
} request_options_t;
|
|
438
|
+
|
|
217
439
|
static int contains_header_injection(const char *str, long len) {
|
|
218
440
|
for (long i = 0; i < len; i++) {
|
|
219
441
|
if (str[i] == '\r' || str[i] == '\n' || str[i] == '\0')
|
|
@@ -222,18 +444,61 @@ static int contains_header_injection(const char *str, long len) {
|
|
|
222
444
|
return 0;
|
|
223
445
|
}
|
|
224
446
|
|
|
225
|
-
|
|
447
|
+
static int is_header_token_char(unsigned char c) {
|
|
448
|
+
if ((c >= 'A' && c <= 'Z') || (c >= 'a' && c <= 'z') || (c >= '0' && c <= '9'))
|
|
449
|
+
return 1;
|
|
450
|
+
switch (c) {
|
|
451
|
+
case '!':
|
|
452
|
+
case '#':
|
|
453
|
+
case '$':
|
|
454
|
+
case '%':
|
|
455
|
+
case '&':
|
|
456
|
+
case '\'':
|
|
457
|
+
case '*':
|
|
458
|
+
case '+':
|
|
459
|
+
case '-':
|
|
460
|
+
case '.':
|
|
461
|
+
case '^':
|
|
462
|
+
case '_':
|
|
463
|
+
case '`':
|
|
464
|
+
case '|':
|
|
465
|
+
case '~':
|
|
466
|
+
return 1;
|
|
467
|
+
default:
|
|
468
|
+
return 0;
|
|
469
|
+
}
|
|
470
|
+
}
|
|
471
|
+
|
|
472
|
+
static int is_valid_header_name(const char *str, long len) {
|
|
473
|
+
if (len <= 0)
|
|
474
|
+
return 0;
|
|
475
|
+
for (long i = 0; i < len; i++) {
|
|
476
|
+
if (!is_header_token_char((unsigned char)str[i]))
|
|
477
|
+
return 0;
|
|
478
|
+
}
|
|
479
|
+
return 1;
|
|
480
|
+
}
|
|
481
|
+
|
|
482
|
+
static VALUE hash_aref_symbol_or_string(VALUE hash, VALUE sym, ID id) {
|
|
483
|
+
VALUE value = rb_hash_aref(hash, sym);
|
|
484
|
+
if (!NIL_P(value))
|
|
485
|
+
return value;
|
|
486
|
+
|
|
487
|
+
const char *name = rb_id2name(id);
|
|
488
|
+
return name ? rb_hash_aref(hash, rb_str_new_cstr(name)) : Qnil;
|
|
489
|
+
}
|
|
490
|
+
|
|
491
|
+
static inline VALUE hash_aref_key(VALUE hash, key_id_t key) {
|
|
492
|
+
return hash_aref_symbol_or_string(hash, fast_syms[key], fast_ids[key]);
|
|
493
|
+
}
|
|
494
|
+
|
|
495
|
+
#ifdef FAST_CURL_HAVE_FIBER_SCHEDULER
|
|
226
496
|
static VALUE current_fiber_scheduler(void) {
|
|
227
497
|
VALUE sched = rb_fiber_scheduler_current();
|
|
228
498
|
if (sched == Qnil || sched == Qfalse)
|
|
229
499
|
return Qnil;
|
|
230
500
|
return sched;
|
|
231
501
|
}
|
|
232
|
-
#else
|
|
233
|
-
static VALUE current_fiber_scheduler(void) {
|
|
234
|
-
return Qnil;
|
|
235
|
-
}
|
|
236
|
-
#endif
|
|
237
502
|
|
|
238
503
|
typedef struct {
|
|
239
504
|
void *(*func)(void *);
|
|
@@ -268,49 +533,100 @@ static void run_via_fiber_worker(VALUE scheduler, void *(*func)(void *), void *a
|
|
|
268
533
|
rb_fiber_scheduler_block(scheduler, ctx.blocker, Qnil);
|
|
269
534
|
rb_funcall(th, rb_intern("join"), 0);
|
|
270
535
|
}
|
|
536
|
+
#endif
|
|
537
|
+
|
|
538
|
+
static VALUE fast_cHeaders = Qnil;
|
|
539
|
+
static VALUE new_headers_hash(void) {
|
|
540
|
+
if (!NIL_P(fast_cHeaders))
|
|
541
|
+
return rb_obj_alloc(fast_cHeaders);
|
|
542
|
+
return rb_hash_new();
|
|
543
|
+
}
|
|
544
|
+
|
|
545
|
+
static void headers_hash_store(VALUE headers_hash, VALUE key, VALUE val, int always_array) {
|
|
546
|
+
VALUE existing = rb_hash_aref(headers_hash, key);
|
|
547
|
+
|
|
548
|
+
if (NIL_P(existing)) {
|
|
549
|
+
rb_hash_aset(headers_hash, key, always_array ? rb_ary_new_from_args(1, val) : val);
|
|
550
|
+
return;
|
|
551
|
+
}
|
|
552
|
+
|
|
553
|
+
if (RB_TYPE_P(existing, T_ARRAY)) {
|
|
554
|
+
rb_ary_push(existing, val);
|
|
555
|
+
return;
|
|
556
|
+
}
|
|
557
|
+
|
|
558
|
+
VALUE joined = rb_str_dup(existing);
|
|
559
|
+
rb_str_cat_cstr(joined, ", ");
|
|
560
|
+
rb_str_append(joined, val);
|
|
561
|
+
rb_hash_aset(headers_hash, key, joined);
|
|
562
|
+
}
|
|
271
563
|
|
|
272
564
|
static VALUE build_response(request_ctx_t *ctx) {
|
|
273
565
|
long status = 0;
|
|
274
566
|
curl_easy_getinfo(ctx->easy, CURLINFO_RESPONSE_CODE, &status);
|
|
275
|
-
|
|
567
|
+
|
|
568
|
+
VALUE headers_hash = new_headers_hash();
|
|
276
569
|
for (int i = 0; i < ctx->headers.count; i++) {
|
|
277
570
|
const char *hdr = ctx->headers.entries[i].str;
|
|
278
571
|
size_t hdr_len = ctx->headers.entries[i].len;
|
|
279
572
|
const char *colon = memchr(hdr, ':', hdr_len);
|
|
280
573
|
if (!colon)
|
|
281
574
|
continue;
|
|
575
|
+
|
|
282
576
|
VALUE key = rb_str_new(hdr, colon - hdr);
|
|
283
|
-
|
|
577
|
+
char *kp = RSTRING_PTR(key);
|
|
578
|
+
long klen = RSTRING_LEN(key);
|
|
579
|
+
for (long k = 0; k < klen; k++)
|
|
580
|
+
kp[k] = (char)tolower((unsigned char)kp[k]);
|
|
581
|
+
int always_array = (klen == 10 && memcmp(kp, "set-cookie", 10) == 0);
|
|
582
|
+
|
|
583
|
+
const char *vs = colon + 1;
|
|
584
|
+
const char *ve = hdr + hdr_len;
|
|
585
|
+
|
|
284
586
|
while (vs < ve && (*vs == ' ' || *vs == '\t'))
|
|
285
587
|
vs++;
|
|
286
588
|
while (ve > vs && (*(ve - 1) == ' ' || *(ve - 1) == '\t'))
|
|
287
589
|
ve--;
|
|
590
|
+
|
|
288
591
|
VALUE val = rb_str_new(vs, ve - vs);
|
|
289
|
-
|
|
592
|
+
headers_hash_store(headers_hash, key, val, always_array);
|
|
290
593
|
}
|
|
594
|
+
|
|
291
595
|
VALUE body_str =
|
|
292
596
|
ctx->body.data ? rb_str_new(ctx->body.data, ctx->body.len) : rb_str_new_cstr("");
|
|
597
|
+
|
|
293
598
|
VALUE result = rb_hash_new();
|
|
294
|
-
rb_hash_aset(result,
|
|
295
|
-
rb_hash_aset(result,
|
|
296
|
-
rb_hash_aset(result,
|
|
599
|
+
rb_hash_aset(result, SYM(KEY_STATUS), LONG2NUM(status));
|
|
600
|
+
rb_hash_aset(result, SYM(KEY_HEADERS), headers_hash);
|
|
601
|
+
rb_hash_aset(result, SYM(KEY_BODY), body_str);
|
|
602
|
+
rb_hash_aset(result, SYM(KEY_ERROR), Qnil);
|
|
603
|
+
rb_hash_aset(result, SYM(KEY_ERROR_CODE), Qnil);
|
|
604
|
+
rb_hash_aset(result, SYM(KEY_ATTEMPTS), INT2NUM(ctx->attempts));
|
|
605
|
+
|
|
606
|
+
char *effective = NULL;
|
|
607
|
+
if (curl_easy_getinfo(ctx->easy, CURLINFO_EFFECTIVE_URL, &effective) == CURLE_OK && effective)
|
|
608
|
+
rb_hash_aset(result, SYM(KEY_EFFECTIVE_URL), rb_str_new_cstr(effective));
|
|
609
|
+
else
|
|
610
|
+
rb_hash_aset(result, SYM(KEY_EFFECTIVE_URL), Qnil);
|
|
611
|
+
|
|
297
612
|
return result;
|
|
298
613
|
}
|
|
299
614
|
|
|
300
|
-
static VALUE build_error_response(const char *message) {
|
|
615
|
+
static VALUE build_error_response(const char *message, error_kind_t kind, int attempts) {
|
|
301
616
|
VALUE r = rb_hash_new();
|
|
302
|
-
rb_hash_aset(r,
|
|
303
|
-
rb_hash_aset(r,
|
|
304
|
-
rb_hash_aset(r,
|
|
617
|
+
rb_hash_aset(r, SYM(KEY_STATUS), INT2NUM(0));
|
|
618
|
+
rb_hash_aset(r, SYM(KEY_HEADERS), Qnil);
|
|
619
|
+
rb_hash_aset(r, SYM(KEY_BODY), rb_str_new_cstr(message));
|
|
620
|
+
rb_hash_aset(r, SYM(KEY_ERROR), error_syms[kind]);
|
|
621
|
+
rb_hash_aset(r, SYM(KEY_ERROR_CODE), Qnil);
|
|
622
|
+
rb_hash_aset(r, SYM(KEY_EFFECTIVE_URL), Qnil);
|
|
623
|
+
rb_hash_aset(r, SYM(KEY_ATTEMPTS), INT2NUM(attempts));
|
|
305
624
|
return r;
|
|
306
625
|
}
|
|
307
626
|
|
|
308
|
-
static VALUE build_error_response_with_code(const char *message, int error_code) {
|
|
309
|
-
VALUE r =
|
|
310
|
-
rb_hash_aset(r,
|
|
311
|
-
rb_hash_aset(r, sym_headers, Qnil);
|
|
312
|
-
rb_hash_aset(r, sym_body, rb_str_new_cstr(message));
|
|
313
|
-
rb_hash_aset(r, sym_error_code, INT2NUM(error_code));
|
|
627
|
+
static VALUE build_error_response_with_code(const char *message, int error_code, int attempts) {
|
|
628
|
+
VALUE r = build_error_response(message, ERR_CURL, attempts);
|
|
629
|
+
rb_hash_aset(r, SYM(KEY_ERROR_CODE), INT2NUM(error_code));
|
|
314
630
|
return r;
|
|
315
631
|
}
|
|
316
632
|
|
|
@@ -334,27 +650,40 @@ static int is_valid_url(const char *url) {
|
|
|
334
650
|
return _r; \
|
|
335
651
|
} while (0)
|
|
336
652
|
|
|
337
|
-
static CURLcode setup_basic_options(CURL *easy, const char *url_str,
|
|
653
|
+
static CURLcode setup_basic_options(CURL *easy, const char *url_str, const request_options_t *opts,
|
|
338
654
|
request_ctx_t *ctx) {
|
|
655
|
+
long timeout_ms = opts->timeout_sec * 1000L;
|
|
656
|
+
|
|
657
|
+
if (opts->deadline_ms > 0) {
|
|
658
|
+
long long remaining = opts->deadline_ms - fast_now_ms();
|
|
659
|
+
if (remaining < 1)
|
|
660
|
+
remaining = 1;
|
|
661
|
+
if (remaining < (long long)timeout_ms)
|
|
662
|
+
timeout_ms = (long)remaining;
|
|
663
|
+
}
|
|
664
|
+
|
|
339
665
|
CURL_SETOPT_CHECK(easy, CURLOPT_URL, url_str);
|
|
340
666
|
CURL_SETOPT_CHECK(easy, CURLOPT_WRITEFUNCTION, write_callback);
|
|
341
667
|
CURL_SETOPT_CHECK(easy, CURLOPT_WRITEDATA, &ctx->body);
|
|
342
668
|
CURL_SETOPT_CHECK(easy, CURLOPT_HEADERFUNCTION, header_callback);
|
|
343
669
|
CURL_SETOPT_CHECK(easy, CURLOPT_HEADERDATA, &ctx->headers);
|
|
344
|
-
CURL_SETOPT_CHECK(easy,
|
|
670
|
+
CURL_SETOPT_CHECK(easy, CURLOPT_TIMEOUT_MS, timeout_ms);
|
|
671
|
+
CURL_SETOPT_CHECK(easy, CURLOPT_CONNECTTIMEOUT_MS, opts->connect_timeout_ms);
|
|
345
672
|
CURL_SETOPT_CHECK(easy, CURLOPT_NOSIGNAL, 1L);
|
|
346
|
-
CURL_SETOPT_CHECK(easy, CURLOPT_FOLLOWLOCATION,
|
|
347
|
-
CURL_SETOPT_CHECK(easy, CURLOPT_MAXREDIRS,
|
|
673
|
+
CURL_SETOPT_CHECK(easy, CURLOPT_FOLLOWLOCATION, opts->follow_redirects);
|
|
674
|
+
CURL_SETOPT_CHECK(easy, CURLOPT_MAXREDIRS, opts->max_redirects);
|
|
348
675
|
CURL_SETOPT_CHECK(easy, CURLOPT_ACCEPT_ENCODING, "");
|
|
349
676
|
CURL_SETOPT_CHECK(easy, CURLOPT_PRIVATE, (char *)ctx);
|
|
350
677
|
CURL_SETOPT_CHECK(easy, CURLOPT_HTTP_VERSION, CURL_HTTP_VERSION_2TLS);
|
|
678
|
+
if (fast_share)
|
|
679
|
+
CURL_SETOPT_CHECK(easy, CURLOPT_SHARE, fast_share);
|
|
351
680
|
return CURLE_OK;
|
|
352
681
|
}
|
|
353
682
|
|
|
354
683
|
static CURLcode setup_security_options(CURL *easy) {
|
|
355
684
|
CURL_SETOPT_CHECK(easy, CURLOPT_SSL_VERIFYPEER, 1L);
|
|
356
685
|
CURL_SETOPT_CHECK(easy, CURLOPT_SSL_VERIFYHOST, 2L);
|
|
357
|
-
#
|
|
686
|
+
#if LIBCURL_VERSION_NUM >= 0x075500
|
|
358
687
|
CURL_SETOPT_CHECK(easy, CURLOPT_PROTOCOLS_STR, "http,https");
|
|
359
688
|
CURL_SETOPT_CHECK(easy, CURLOPT_REDIR_PROTOCOLS_STR, "http,https");
|
|
360
689
|
#else
|
|
@@ -364,107 +693,231 @@ static CURLcode setup_security_options(CURL *easy) {
|
|
|
364
693
|
return CURLE_OK;
|
|
365
694
|
}
|
|
366
695
|
|
|
367
|
-
static CURLcode
|
|
368
|
-
|
|
369
|
-
|
|
370
|
-
|
|
371
|
-
|
|
372
|
-
|
|
373
|
-
|
|
374
|
-
|
|
375
|
-
|
|
376
|
-
|
|
377
|
-
|
|
378
|
-
|
|
379
|
-
|
|
696
|
+
static CURLcode set_body(CURL *easy, VALUE body) {
|
|
697
|
+
VALUE body_str = rb_String(body);
|
|
698
|
+
CURL_SETOPT_CHECK(easy, CURLOPT_POSTFIELDSIZE, (long)RSTRING_LEN(body_str));
|
|
699
|
+
CURL_SETOPT_CHECK(easy, CURLOPT_COPYPOSTFIELDS, StringValuePtr(body_str));
|
|
700
|
+
RB_GC_GUARD(body_str);
|
|
701
|
+
return CURLE_OK;
|
|
702
|
+
}
|
|
703
|
+
|
|
704
|
+
typedef struct {
|
|
705
|
+
const char *name;
|
|
706
|
+
const char *custom;
|
|
707
|
+
int post;
|
|
708
|
+
int nobody;
|
|
709
|
+
int allows_body;
|
|
710
|
+
int idempotent;
|
|
711
|
+
} http_method_t;
|
|
712
|
+
|
|
713
|
+
static const http_method_t HTTP_METHODS[] = {
|
|
714
|
+
{"GET", NULL, 0, 0, 0, 1}, {"POST", NULL, 1, 0, 1, 0}, {"PUT", "PUT", 0, 0, 1, 1},
|
|
715
|
+
{"DELETE", "DELETE", 0, 0, 1, 1}, {"PATCH", "PATCH", 0, 0, 1, 0}, {"HEAD", NULL, 0, 1, 0, 1},
|
|
716
|
+
{"OPTIONS", "OPTIONS", 0, 0, 0, 1},
|
|
717
|
+
};
|
|
718
|
+
|
|
719
|
+
static const http_method_t *find_http_method(const char *name) {
|
|
720
|
+
for (size_t i = 0; i < sizeof(HTTP_METHODS) / sizeof(HTTP_METHODS[0]); i++) {
|
|
721
|
+
if (strcasecmp(HTTP_METHODS[i].name, name) == 0)
|
|
722
|
+
return &HTTP_METHODS[i];
|
|
723
|
+
}
|
|
724
|
+
return NULL;
|
|
725
|
+
}
|
|
726
|
+
|
|
727
|
+
static CURLcode apply_http_method(CURL *easy, const http_method_t *method) {
|
|
728
|
+
if (method->post)
|
|
729
|
+
CURL_SETOPT_CHECK(easy, CURLOPT_POST, 1L);
|
|
730
|
+
if (method->custom)
|
|
731
|
+
CURL_SETOPT_CHECK(easy, CURLOPT_CUSTOMREQUEST, method->custom);
|
|
732
|
+
if (method->nobody)
|
|
733
|
+
CURL_SETOPT_CHECK(easy, CURLOPT_NOBODY, 1L);
|
|
734
|
+
return CURLE_OK;
|
|
735
|
+
}
|
|
736
|
+
|
|
737
|
+
static int setup_method_and_body(request_ctx_t *ctx, VALUE method_value, VALUE body) {
|
|
738
|
+
const http_method_t *method;
|
|
739
|
+
int has_body = !NIL_P(body);
|
|
740
|
+
char name[32];
|
|
741
|
+
|
|
742
|
+
if (NIL_P(method_value)) {
|
|
743
|
+
memcpy(name, "GET", 4);
|
|
744
|
+
} else {
|
|
745
|
+
if (!RB_TYPE_P(method_value, T_STRING)) {
|
|
746
|
+
SET_SETUP_ERROR(ctx, "Unsupported HTTP method", 1);
|
|
747
|
+
return 0;
|
|
380
748
|
}
|
|
749
|
+
long len = RSTRING_LEN(method_value);
|
|
750
|
+
if (len <= 0 || len >= (long)sizeof(name)) {
|
|
751
|
+
SET_SETUP_ERROR(ctx, "Unsupported HTTP method", 1);
|
|
752
|
+
return 0;
|
|
753
|
+
}
|
|
754
|
+
memcpy(name, RSTRING_PTR(method_value), (size_t)len);
|
|
755
|
+
name[len] = '\0';
|
|
756
|
+
}
|
|
757
|
+
|
|
758
|
+
method = find_http_method(name);
|
|
759
|
+
if (!method) {
|
|
760
|
+
SET_SETUP_ERROR(ctx, "Unsupported HTTP method", 1);
|
|
761
|
+
return 0;
|
|
762
|
+
}
|
|
763
|
+
if (has_body && !method->allows_body) {
|
|
764
|
+
SET_SETUP_ERROR(ctx, "This HTTP method must not include a body", 1);
|
|
765
|
+
return 0;
|
|
766
|
+
}
|
|
767
|
+
|
|
768
|
+
ctx->idempotent = method->idempotent;
|
|
769
|
+
|
|
770
|
+
if (apply_http_method(ctx->easy, method) != CURLE_OK) {
|
|
771
|
+
SET_SETUP_ERROR(ctx, "Failed to configure HTTP method", 0);
|
|
772
|
+
return 0;
|
|
381
773
|
}
|
|
382
774
|
|
|
383
|
-
if (
|
|
384
|
-
|
|
385
|
-
|
|
775
|
+
if (has_body && set_body(ctx->easy, body) != CURLE_OK) {
|
|
776
|
+
SET_SETUP_ERROR(ctx, "Failed to configure request body", 0);
|
|
777
|
+
return 0;
|
|
386
778
|
}
|
|
387
|
-
|
|
779
|
+
|
|
780
|
+
RB_GC_GUARD(method_value);
|
|
781
|
+
RB_GC_GUARD(body);
|
|
782
|
+
return 1;
|
|
783
|
+
}
|
|
784
|
+
|
|
785
|
+
static void append_request_header(request_ctx_t *ctx, const char *buf) {
|
|
786
|
+
struct curl_slist *new_headers = curl_slist_append(ctx->req_headers, buf);
|
|
787
|
+
if (!new_headers)
|
|
788
|
+
rb_raise(rb_eNoMemError, "failed to allocate request header");
|
|
789
|
+
ctx->req_headers = new_headers;
|
|
790
|
+
}
|
|
791
|
+
|
|
792
|
+
static char *alloc_header_line(const char *key, long key_len, const char *value, long value_len,
|
|
793
|
+
char stack_buf[HEADER_LINE_BUF_SIZE]) {
|
|
794
|
+
int has_value = value && value_len > 0;
|
|
795
|
+
long need = key_len + (has_value ? 2 + value_len : 1) + 1;
|
|
796
|
+
char *buf = need > HEADER_LINE_BUF_SIZE ? malloc((size_t)need) : stack_buf;
|
|
797
|
+
|
|
798
|
+
if (!buf)
|
|
799
|
+
rb_raise(rb_eNoMemError, "failed to allocate request header");
|
|
800
|
+
|
|
801
|
+
memcpy(buf, key, (size_t)key_len);
|
|
802
|
+
if (has_value) {
|
|
803
|
+
buf[key_len] = ':';
|
|
804
|
+
buf[key_len + 1] = ' ';
|
|
805
|
+
memcpy(buf + key_len + 2, value, (size_t)value_len);
|
|
806
|
+
buf[key_len + 2 + value_len] = '\0';
|
|
807
|
+
} else {
|
|
808
|
+
buf[key_len] = ';';
|
|
809
|
+
buf[key_len + 1] = '\0';
|
|
810
|
+
}
|
|
811
|
+
|
|
812
|
+
return buf;
|
|
813
|
+
}
|
|
814
|
+
|
|
815
|
+
static void append_formatted_header(request_ctx_t *ctx, const char *key, long key_len,
|
|
816
|
+
const char *value, long value_len) {
|
|
817
|
+
char stack_buf[HEADER_LINE_BUF_SIZE];
|
|
818
|
+
char *line = alloc_header_line(key, key_len, value, value_len, stack_buf);
|
|
819
|
+
append_request_header(ctx, line);
|
|
820
|
+
if (line != stack_buf)
|
|
821
|
+
free(line);
|
|
822
|
+
}
|
|
823
|
+
|
|
824
|
+
static VALUE header_part_to_str(VALUE v) {
|
|
825
|
+
if (RB_TYPE_P(v, T_STRING))
|
|
826
|
+
return v;
|
|
827
|
+
if (SYMBOL_P(v))
|
|
828
|
+
return rb_sym2str(v);
|
|
829
|
+
return rb_String(v);
|
|
388
830
|
}
|
|
389
831
|
|
|
390
832
|
static int header_iter_cb(VALUE key, VALUE val, VALUE arg) {
|
|
391
833
|
request_ctx_t *ctx = (request_ctx_t *)arg;
|
|
392
|
-
VALUE key_str
|
|
393
|
-
const char *k
|
|
394
|
-
long klen
|
|
395
|
-
|
|
396
|
-
|
|
397
|
-
|
|
398
|
-
|
|
399
|
-
|
|
400
|
-
|
|
401
|
-
|
|
402
|
-
|
|
403
|
-
|
|
404
|
-
|
|
405
|
-
|
|
406
|
-
|
|
407
|
-
|
|
408
|
-
|
|
409
|
-
|
|
410
|
-
|
|
411
|
-
|
|
412
|
-
|
|
413
|
-
|
|
414
|
-
|
|
415
|
-
|
|
416
|
-
|
|
417
|
-
|
|
418
|
-
if (contains_header_injection(v, vlen))
|
|
419
|
-
return ST_CONTINUE;
|
|
420
|
-
char sbuf[HEADER_LINE_BUF_SIZE];
|
|
421
|
-
char *buf = sbuf;
|
|
422
|
-
long need = klen + 2 + vlen + 1;
|
|
423
|
-
if (need > HEADER_LINE_BUF_SIZE)
|
|
424
|
-
buf = malloc(need);
|
|
425
|
-
if (!buf)
|
|
426
|
-
return ST_CONTINUE;
|
|
427
|
-
|
|
428
|
-
memcpy(buf, k, klen);
|
|
429
|
-
buf[klen] = ':';
|
|
430
|
-
buf[klen + 1] = ' ';
|
|
431
|
-
memcpy(buf + klen + 2, v, vlen);
|
|
432
|
-
buf[klen + 2 + vlen] = '\0';
|
|
433
|
-
ctx->req_headers = curl_slist_append(ctx->req_headers, buf);
|
|
434
|
-
if (buf != sbuf)
|
|
435
|
-
free(buf);
|
|
834
|
+
VALUE key_str, val_str;
|
|
835
|
+
const char *k;
|
|
836
|
+
long klen;
|
|
837
|
+
const char *v = NULL;
|
|
838
|
+
long vlen = 0;
|
|
839
|
+
|
|
840
|
+
if (ctx->setup_error)
|
|
841
|
+
return ST_STOP;
|
|
842
|
+
|
|
843
|
+
key_str = header_part_to_str(key);
|
|
844
|
+
val_str = NIL_P(val) ? Qnil : header_part_to_str(val);
|
|
845
|
+
k = RSTRING_PTR(key_str);
|
|
846
|
+
klen = RSTRING_LEN(key_str);
|
|
847
|
+
|
|
848
|
+
if (!is_valid_header_name(k, klen) || contains_header_injection(k, klen)) {
|
|
849
|
+
SET_SETUP_ERROR(ctx, "Invalid HTTP header name", 1);
|
|
850
|
+
return ST_STOP;
|
|
851
|
+
}
|
|
852
|
+
|
|
853
|
+
if (!NIL_P(val_str) && RSTRING_LEN(val_str) > 0) {
|
|
854
|
+
v = RSTRING_PTR(val_str);
|
|
855
|
+
vlen = RSTRING_LEN(val_str);
|
|
856
|
+
if (contains_header_injection(v, vlen)) {
|
|
857
|
+
SET_SETUP_ERROR(ctx, "Invalid HTTP header value", 1);
|
|
858
|
+
return ST_STOP;
|
|
859
|
+
}
|
|
436
860
|
}
|
|
861
|
+
|
|
862
|
+
append_formatted_header(ctx, k, klen, v, vlen);
|
|
863
|
+
|
|
864
|
+
RB_GC_GUARD(key_str);
|
|
865
|
+
RB_GC_GUARD(val_str);
|
|
437
866
|
return ST_CONTINUE;
|
|
438
867
|
}
|
|
439
868
|
|
|
440
|
-
static int setup_easy_handle(request_ctx_t *ctx, VALUE request,
|
|
441
|
-
|
|
442
|
-
|
|
443
|
-
|
|
444
|
-
|
|
445
|
-
|
|
869
|
+
static int setup_easy_handle(request_ctx_t *ctx, VALUE request, const request_options_t *opts) {
|
|
870
|
+
if (!RB_TYPE_P(request, T_HASH)) {
|
|
871
|
+
SET_SETUP_ERROR(ctx, "Request must be a Hash", 1);
|
|
872
|
+
return 0;
|
|
873
|
+
}
|
|
874
|
+
|
|
875
|
+
VALUE url = hash_aref_key(request, KEY_URL);
|
|
876
|
+
VALUE method = hash_aref_key(request, KEY_METHOD);
|
|
877
|
+
VALUE headers = hash_aref_key(request, KEY_HEADERS);
|
|
878
|
+
VALUE body = hash_aref_key(request, KEY_BODY);
|
|
879
|
+
|
|
880
|
+
if (NIL_P(url)) {
|
|
881
|
+
SET_SETUP_ERROR(ctx, "Missing :url", 0);
|
|
882
|
+
return 0;
|
|
883
|
+
}
|
|
884
|
+
if (!RB_TYPE_P(url, T_STRING) || memchr(RSTRING_PTR(url), '\0', RSTRING_LEN(url))) {
|
|
885
|
+
SET_SETUP_ERROR(ctx, "Invalid URL", 1);
|
|
446
886
|
return 0;
|
|
887
|
+
}
|
|
888
|
+
|
|
447
889
|
const char *url_str = StringValueCStr(url);
|
|
448
|
-
if (!is_valid_url(url_str))
|
|
449
|
-
|
|
890
|
+
if (!is_valid_url(url_str)) {
|
|
891
|
+
SET_SETUP_ERROR(ctx, "Invalid URL (expected http:// or https://, max 2048 bytes)", 1);
|
|
892
|
+
return 0;
|
|
893
|
+
}
|
|
450
894
|
|
|
451
|
-
|
|
452
|
-
|
|
453
|
-
if (res != CURLE_OK)
|
|
895
|
+
if (setup_basic_options(ctx->easy, url_str, opts, ctx) != CURLE_OK) {
|
|
896
|
+
SET_SETUP_ERROR(ctx, "Failed to configure request", 0);
|
|
454
897
|
return 0;
|
|
455
|
-
|
|
456
|
-
|
|
898
|
+
}
|
|
899
|
+
|
|
900
|
+
if (setup_security_options(ctx->easy) != CURLE_OK) {
|
|
901
|
+
SET_SETUP_ERROR(ctx, "Failed to configure TLS options", 0);
|
|
457
902
|
return 0;
|
|
458
|
-
|
|
459
|
-
|
|
903
|
+
}
|
|
904
|
+
|
|
905
|
+
if (!setup_method_and_body(ctx, method, body))
|
|
460
906
|
return 0;
|
|
461
907
|
|
|
462
|
-
if (!NIL_P(headers)
|
|
908
|
+
if (!NIL_P(headers)) {
|
|
909
|
+
if (!RB_TYPE_P(headers, T_HASH)) {
|
|
910
|
+
SET_SETUP_ERROR(ctx, ":headers must be a Hash", 1);
|
|
911
|
+
return 0;
|
|
912
|
+
}
|
|
463
913
|
rb_hash_foreach(headers, header_iter_cb, (VALUE)ctx);
|
|
914
|
+
if (ctx->setup_error)
|
|
915
|
+
return 0;
|
|
464
916
|
if (ctx->req_headers) {
|
|
465
|
-
|
|
466
|
-
|
|
917
|
+
if (curl_easy_setopt(ctx->easy, CURLOPT_HTTPHEADER, ctx->req_headers) != CURLE_OK) {
|
|
918
|
+
SET_SETUP_ERROR(ctx, "Failed to set request headers", 0);
|
|
467
919
|
return 0;
|
|
920
|
+
}
|
|
468
921
|
}
|
|
469
922
|
}
|
|
470
923
|
|
|
@@ -476,13 +929,25 @@ static int setup_easy_handle(request_ctx_t *ctx, VALUE request, long timeout_sec
|
|
|
476
929
|
return 1;
|
|
477
930
|
}
|
|
478
931
|
|
|
932
|
+
/* Poll until something actually completes, or the slice budget expires.
|
|
933
|
+
Previously this returned after every single 50ms poll, which under a fiber
|
|
934
|
+
scheduler meant a fresh OS thread twenty times a second. Cancellation still
|
|
935
|
+
breaks out immediately via curl_multi_wakeup. */
|
|
479
936
|
static void *poll_without_gvl(void *arg) {
|
|
480
937
|
multi_session_t *s = (multi_session_t *)arg;
|
|
481
|
-
|
|
482
|
-
|
|
483
|
-
|
|
484
|
-
|
|
485
|
-
|
|
938
|
+
long long started = fast_now_ms();
|
|
939
|
+
int before = s->still_running;
|
|
940
|
+
|
|
941
|
+
while (!s->cancelled) {
|
|
942
|
+
int numfds = 0;
|
|
943
|
+
curl_multi_poll(s->multi, NULL, 0, POLL_TIMEOUT_MS, &numfds);
|
|
944
|
+
curl_multi_perform(s->multi, &s->still_running);
|
|
945
|
+
|
|
946
|
+
if (s->still_running == 0 || s->still_running < before)
|
|
947
|
+
break;
|
|
948
|
+
if (fast_now_ms() - started >= POLL_SLICE_MS)
|
|
949
|
+
break;
|
|
950
|
+
}
|
|
486
951
|
return NULL;
|
|
487
952
|
}
|
|
488
953
|
|
|
@@ -501,61 +966,185 @@ typedef struct {
|
|
|
501
966
|
int stream;
|
|
502
967
|
} completion_ctx_t;
|
|
503
968
|
|
|
969
|
+
static VALUE build_result_pair(int index, VALUE response) {
|
|
970
|
+
return rb_ary_new_from_args(2, INT2NUM(index), response);
|
|
971
|
+
}
|
|
972
|
+
|
|
973
|
+
static int record_immediate_error(completion_ctx_t *cctx, int index, const char *message,
|
|
974
|
+
int attempts) {
|
|
975
|
+
if (cctx->stream || cctx->target > 0) {
|
|
976
|
+
VALUE pair =
|
|
977
|
+
build_result_pair(index, build_error_response(message, ERR_INVALID_REQUEST, attempts));
|
|
978
|
+
|
|
979
|
+
if (cctx->stream)
|
|
980
|
+
rb_yield(pair);
|
|
981
|
+
else
|
|
982
|
+
rb_ary_push(cctx->results, pair);
|
|
983
|
+
|
|
984
|
+
cctx->completed++;
|
|
985
|
+
if (cctx->target > 0 && cctx->completed >= cctx->target)
|
|
986
|
+
return 1;
|
|
987
|
+
}
|
|
988
|
+
return 0;
|
|
989
|
+
}
|
|
990
|
+
|
|
504
991
|
static int process_completed(multi_session_t *session, completion_ctx_t *cctx) {
|
|
505
992
|
CURLMsg *msg;
|
|
506
993
|
int msgs_left;
|
|
994
|
+
|
|
507
995
|
while ((msg = curl_multi_info_read(session->multi, &msgs_left))) {
|
|
508
996
|
if (msg->msg != CURLMSG_DONE)
|
|
509
997
|
continue;
|
|
998
|
+
|
|
510
999
|
request_ctx_t *ctx = NULL;
|
|
511
1000
|
curl_easy_getinfo(msg->easy_handle, CURLINFO_PRIVATE, (char **)&ctx);
|
|
512
1001
|
if (!ctx || ctx->done)
|
|
513
1002
|
continue;
|
|
1003
|
+
|
|
1004
|
+
if (ctx->active) {
|
|
1005
|
+
curl_multi_remove_handle(session->multi, ctx->easy);
|
|
1006
|
+
ctx->active = 0;
|
|
1007
|
+
if (session->active_count > 0)
|
|
1008
|
+
session->active_count--;
|
|
1009
|
+
}
|
|
1010
|
+
|
|
514
1011
|
ctx->done = 1;
|
|
515
1012
|
ctx->curl_result = msg->data.result;
|
|
516
1013
|
if (msg->data.result == CURLE_OK)
|
|
517
1014
|
curl_easy_getinfo(ctx->easy, CURLINFO_RESPONSE_CODE, &ctx->http_status);
|
|
518
|
-
|
|
519
|
-
|
|
520
|
-
|
|
521
|
-
|
|
522
|
-
|
|
523
|
-
|
|
524
|
-
|
|
525
|
-
|
|
526
|
-
|
|
527
|
-
cctx->
|
|
1015
|
+
|
|
1016
|
+
if (cctx->stream || cctx->target > 0) {
|
|
1017
|
+
VALUE response =
|
|
1018
|
+
(msg->data.result == CURLE_OK)
|
|
1019
|
+
? build_response(ctx)
|
|
1020
|
+
: build_error_response_with_code(curl_easy_strerror(msg->data.result),
|
|
1021
|
+
(int)msg->data.result, ctx->attempts);
|
|
1022
|
+
VALUE pair = build_result_pair(ctx->index, response);
|
|
1023
|
+
|
|
1024
|
+
if (cctx->stream)
|
|
1025
|
+
rb_yield(pair);
|
|
1026
|
+
else
|
|
1027
|
+
rb_ary_push(cctx->results, pair);
|
|
528
1028
|
}
|
|
1029
|
+
|
|
1030
|
+
cctx->completed++;
|
|
529
1031
|
if (cctx->target > 0 && cctx->completed >= cctx->target)
|
|
530
1032
|
return 1;
|
|
531
1033
|
}
|
|
1034
|
+
|
|
1035
|
+
return 0;
|
|
1036
|
+
}
|
|
1037
|
+
|
|
1038
|
+
static int next_pending_index(multi_session_t *session) {
|
|
1039
|
+
if (session->pending_pos >= session->pending_count)
|
|
1040
|
+
return -1;
|
|
1041
|
+
|
|
1042
|
+
if (session->pending_indices)
|
|
1043
|
+
return session->pending_indices[session->pending_pos++];
|
|
1044
|
+
|
|
1045
|
+
return session->pending_pos++;
|
|
1046
|
+
}
|
|
1047
|
+
|
|
1048
|
+
static int activate_request(multi_session_t *session, VALUE requests, int idx, int *invalid,
|
|
1049
|
+
const request_options_t *opts) {
|
|
1050
|
+
request_ctx_t *ctx = &session->requests[idx];
|
|
1051
|
+
|
|
1052
|
+
if (invalid[idx] || ctx->done)
|
|
1053
|
+
return 0;
|
|
1054
|
+
|
|
1055
|
+
if (!request_ctx_prepare_easy(ctx)) {
|
|
1056
|
+
SET_SETUP_ERROR(ctx, "Failed to allocate a curl handle", 0);
|
|
1057
|
+
return 0;
|
|
1058
|
+
}
|
|
1059
|
+
|
|
1060
|
+
if (!setup_easy_handle(ctx, rb_ary_entry(requests, idx), opts))
|
|
1061
|
+
return 0;
|
|
1062
|
+
|
|
1063
|
+
if (curl_multi_add_handle(session->multi, ctx->easy) != CURLM_OK) {
|
|
1064
|
+
SET_SETUP_ERROR(ctx, "Failed to schedule the request", 0);
|
|
1065
|
+
return 0;
|
|
1066
|
+
}
|
|
1067
|
+
|
|
1068
|
+
ctx->attempts++;
|
|
1069
|
+
ctx->active = 1;
|
|
1070
|
+
ctx->done = 0;
|
|
1071
|
+
session->active_count++;
|
|
1072
|
+
return 1;
|
|
1073
|
+
}
|
|
1074
|
+
|
|
1075
|
+
static int fill_slots(multi_session_t *session, VALUE requests, int *invalid,
|
|
1076
|
+
const request_options_t *opts, completion_ctx_t *cctx) {
|
|
1077
|
+
while (session->active_count < session->max_connections) {
|
|
1078
|
+
int idx = next_pending_index(session);
|
|
1079
|
+
if (idx < 0)
|
|
1080
|
+
break;
|
|
1081
|
+
|
|
1082
|
+
request_ctx_t *ctx = &session->requests[idx];
|
|
1083
|
+
|
|
1084
|
+
if (!activate_request(session, requests, idx, invalid, opts)) {
|
|
1085
|
+
invalid[idx] = 1;
|
|
1086
|
+
ctx->done = 1;
|
|
1087
|
+
ctx->active = 0;
|
|
1088
|
+
if (record_immediate_error(cctx, idx,
|
|
1089
|
+
ctx->setup_error ? ctx->setup_error
|
|
1090
|
+
: "Invalid request configuration",
|
|
1091
|
+
ctx->attempts))
|
|
1092
|
+
return 1;
|
|
1093
|
+
}
|
|
1094
|
+
}
|
|
1095
|
+
|
|
532
1096
|
return 0;
|
|
533
1097
|
}
|
|
534
1098
|
|
|
535
|
-
static
|
|
1099
|
+
static int pending_remaining(multi_session_t *session) {
|
|
1100
|
+
return session->pending_pos < session->pending_count;
|
|
1101
|
+
}
|
|
1102
|
+
|
|
1103
|
+
static void prepare_pending(multi_session_t *session, int *indices, int count) {
|
|
1104
|
+
session->pending_indices = indices;
|
|
1105
|
+
session->pending_count = count;
|
|
1106
|
+
session->pending_pos = 0;
|
|
1107
|
+
session->active_count = 0;
|
|
1108
|
+
session->still_running = 0;
|
|
1109
|
+
}
|
|
1110
|
+
|
|
1111
|
+
static void run_multi_loop(multi_session_t *session, completion_ctx_t *cctx, VALUE requests,
|
|
1112
|
+
int *invalid, const request_options_t *opts, int *indices,
|
|
1113
|
+
int indices_count) {
|
|
1114
|
+
#ifdef FAST_CURL_HAVE_FIBER_SCHEDULER
|
|
536
1115
|
VALUE scheduler = current_fiber_scheduler();
|
|
1116
|
+
#endif
|
|
1117
|
+
prepare_pending(session, indices, indices_count);
|
|
537
1118
|
|
|
538
|
-
if (
|
|
539
|
-
|
|
540
|
-
|
|
541
|
-
|
|
542
|
-
|
|
1119
|
+
if (fill_slots(session, requests, invalid, opts, cctx))
|
|
1120
|
+
return;
|
|
1121
|
+
|
|
1122
|
+
curl_multi_perform(session->multi, &session->still_running);
|
|
1123
|
+
if (process_completed(session, cctx))
|
|
1124
|
+
return;
|
|
1125
|
+
|
|
1126
|
+
while (!session->cancelled && (session->active_count > 0 || pending_remaining(session))) {
|
|
1127
|
+
if (opts->deadline_ms > 0 && fast_now_ms() >= opts->deadline_ms)
|
|
1128
|
+
break;
|
|
1129
|
+
|
|
1130
|
+
if (fill_slots(session, requests, invalid, opts, cctx))
|
|
1131
|
+
return;
|
|
1132
|
+
|
|
1133
|
+
if (session->active_count == 0)
|
|
1134
|
+
break;
|
|
1135
|
+
|
|
1136
|
+
#ifdef FAST_CURL_HAVE_FIBER_SCHEDULER
|
|
1137
|
+
if (scheduler != Qnil)
|
|
543
1138
|
run_via_fiber_worker(scheduler, poll_without_gvl, session);
|
|
544
|
-
|
|
545
|
-
|
|
546
|
-
}
|
|
547
|
-
process_completed(session, cctx);
|
|
548
|
-
} else {
|
|
549
|
-
curl_multi_perform(session->multi, &session->still_running);
|
|
550
|
-
while (session->still_running > 0) {
|
|
551
|
-
if (session->cancelled)
|
|
552
|
-
break;
|
|
1139
|
+
else
|
|
1140
|
+
#endif
|
|
553
1141
|
rb_thread_call_without_gvl(poll_without_gvl, session, unblock_perform, session);
|
|
554
|
-
|
|
555
|
-
|
|
556
|
-
|
|
557
|
-
process_completed(session, cctx);
|
|
1142
|
+
|
|
1143
|
+
if (process_completed(session, cctx))
|
|
1144
|
+
return;
|
|
558
1145
|
}
|
|
1146
|
+
|
|
1147
|
+
process_completed(session, cctx);
|
|
559
1148
|
}
|
|
560
1149
|
|
|
561
1150
|
static int is_default_retryable_curle(CURLcode code) {
|
|
@@ -566,11 +1155,16 @@ static int is_default_retryable_curle(CURLcode code) {
|
|
|
566
1155
|
}
|
|
567
1156
|
|
|
568
1157
|
static int should_retry(request_ctx_t *ctx, retry_config_t *rc) {
|
|
1158
|
+
if (!ctx->idempotent && !rc->retry_non_idempotent)
|
|
1159
|
+
return 0;
|
|
1160
|
+
|
|
569
1161
|
if (ctx->curl_result != CURLE_OK)
|
|
570
1162
|
return is_default_retryable_curle(ctx->curl_result);
|
|
1163
|
+
|
|
571
1164
|
for (int i = 0; i < rc->retry_http_count; i++)
|
|
572
1165
|
if (rc->retry_http_codes[i] == (int)ctx->http_status)
|
|
573
1166
|
return 1;
|
|
1167
|
+
|
|
574
1168
|
return 0;
|
|
575
1169
|
}
|
|
576
1170
|
|
|
@@ -586,10 +1180,11 @@ static void *sleep_without_gvl(void *arg) {
|
|
|
586
1180
|
return NULL;
|
|
587
1181
|
}
|
|
588
1182
|
|
|
589
|
-
/* FIX #2: Fiber path releases GVL via run_via_fiber_worker */
|
|
590
1183
|
static void retry_delay_sleep(long delay_ms) {
|
|
591
1184
|
if (delay_ms <= 0)
|
|
592
1185
|
return;
|
|
1186
|
+
|
|
1187
|
+
#ifdef FAST_CURL_HAVE_FIBER_SCHEDULER
|
|
593
1188
|
VALUE scheduler = current_fiber_scheduler();
|
|
594
1189
|
if (scheduler != Qnil) {
|
|
595
1190
|
long remaining = delay_ms;
|
|
@@ -599,70 +1194,156 @@ static void retry_delay_sleep(long delay_ms) {
|
|
|
599
1194
|
run_via_fiber_worker(scheduler, sleep_without_gvl, &sa);
|
|
600
1195
|
remaining -= chunk;
|
|
601
1196
|
}
|
|
602
|
-
} else
|
|
1197
|
+
} else
|
|
1198
|
+
#endif
|
|
1199
|
+
{
|
|
603
1200
|
sleep_arg_t sa = {.delay_ms = delay_ms};
|
|
604
1201
|
rb_thread_call_without_gvl(sleep_without_gvl, &sa, (rb_unblock_function_t *)0, NULL);
|
|
605
1202
|
}
|
|
606
1203
|
}
|
|
607
1204
|
|
|
608
|
-
static void
|
|
609
|
-
*timeout = 30;
|
|
610
|
-
*max_conn = 20;
|
|
1205
|
+
static void retry_config_init(retry_config_t *retry_cfg) {
|
|
611
1206
|
retry_cfg->max_retries = DEFAULT_RETRIES;
|
|
1207
|
+
retry_cfg->retries_explicit = 0;
|
|
612
1208
|
retry_cfg->retry_delay_ms = DEFAULT_RETRY_DELAY;
|
|
613
1209
|
retry_cfg->retry_http_codes = NULL;
|
|
614
1210
|
retry_cfg->retry_http_count = 0;
|
|
615
|
-
|
|
1211
|
+
retry_cfg->retry_non_idempotent = 0;
|
|
1212
|
+
}
|
|
1213
|
+
|
|
1214
|
+
static long retry_backoff_ms(const retry_config_t *rc, int attempt) {
|
|
1215
|
+
long delay = rc->retry_delay_ms;
|
|
1216
|
+
long half;
|
|
1217
|
+
|
|
1218
|
+
if (delay <= 0)
|
|
1219
|
+
return 0;
|
|
1220
|
+
|
|
1221
|
+
for (int i = 0; i < attempt && delay < MAX_RETRY_DELAY_MS; i++)
|
|
1222
|
+
delay *= 2;
|
|
1223
|
+
if (delay > MAX_RETRY_DELAY_MS)
|
|
1224
|
+
delay = MAX_RETRY_DELAY_MS;
|
|
1225
|
+
|
|
1226
|
+
half = delay / 2;
|
|
1227
|
+
if (half > 0)
|
|
1228
|
+
delay = half + (long)(fast_rand() % (uint32_t)(half + 1));
|
|
1229
|
+
|
|
1230
|
+
return delay;
|
|
1231
|
+
}
|
|
1232
|
+
|
|
1233
|
+
static long parse_long_option(VALUE options, key_id_t key, const char *name, long min, long max,
|
|
1234
|
+
long default_value, int *present) {
|
|
1235
|
+
VALUE raw = hash_aref_key(options, key);
|
|
1236
|
+
long value;
|
|
1237
|
+
|
|
1238
|
+
if (present)
|
|
1239
|
+
*present = !NIL_P(raw);
|
|
1240
|
+
if (NIL_P(raw))
|
|
1241
|
+
return default_value;
|
|
1242
|
+
|
|
1243
|
+
value = NUM2LONG(raw);
|
|
1244
|
+
if (value < min || value > max)
|
|
1245
|
+
rb_raise(rb_eArgError, "%s must be between %ld and %ld", name, min, max);
|
|
1246
|
+
|
|
1247
|
+
return value;
|
|
1248
|
+
}
|
|
1249
|
+
|
|
1250
|
+
static int parse_int_option(VALUE options, key_id_t key, const char *name, int min, int max,
|
|
1251
|
+
int default_value, int *present) {
|
|
1252
|
+
return (int)parse_long_option(options, key, name, min, max, default_value, present);
|
|
1253
|
+
}
|
|
1254
|
+
|
|
1255
|
+
static void parse_retry_codes(VALUE options, retry_config_t *retry_cfg) {
|
|
1256
|
+
VALUE codes = hash_aref_key(options, KEY_RETRY_CODES);
|
|
1257
|
+
long len_long;
|
|
1258
|
+
int len;
|
|
1259
|
+
|
|
1260
|
+
if (NIL_P(codes))
|
|
616
1261
|
return;
|
|
617
1262
|
|
|
618
|
-
|
|
619
|
-
|
|
620
|
-
|
|
621
|
-
|
|
622
|
-
|
|
623
|
-
|
|
624
|
-
|
|
625
|
-
|
|
626
|
-
|
|
627
|
-
|
|
628
|
-
if (!NIL_P(c)) {
|
|
629
|
-
int v = NUM2INT(c);
|
|
630
|
-
if (v > 100)
|
|
631
|
-
v = 100;
|
|
632
|
-
else if (v <= 0)
|
|
633
|
-
v = 20;
|
|
634
|
-
*max_conn = v;
|
|
635
|
-
}
|
|
636
|
-
VALUE r = rb_hash_aref(options, sym_retries);
|
|
637
|
-
if (!NIL_P(r)) {
|
|
638
|
-
int v = NUM2INT(r);
|
|
639
|
-
if (v < 0)
|
|
640
|
-
v = 0;
|
|
641
|
-
if (v > MAX_RETRIES)
|
|
642
|
-
v = MAX_RETRIES;
|
|
643
|
-
retry_cfg->max_retries = v;
|
|
644
|
-
}
|
|
645
|
-
VALUE rd = rb_hash_aref(options, sym_retry_delay);
|
|
646
|
-
if (!NIL_P(rd)) {
|
|
647
|
-
long v = NUM2LONG(rd);
|
|
648
|
-
if (v < 0)
|
|
649
|
-
v = 0;
|
|
650
|
-
if (v > 30000)
|
|
651
|
-
v = 30000;
|
|
652
|
-
retry_cfg->retry_delay_ms = v;
|
|
653
|
-
}
|
|
654
|
-
VALUE rc = rb_hash_aref(options, sym_retry_codes);
|
|
655
|
-
if (!NIL_P(rc) && rb_obj_is_kind_of(rc, rb_cArray)) {
|
|
656
|
-
int len = (int)RARRAY_LEN(rc);
|
|
657
|
-
if (len > 0) {
|
|
658
|
-
retry_cfg->retry_http_codes = malloc(sizeof(int) * len);
|
|
659
|
-
if (retry_cfg->retry_http_codes) {
|
|
660
|
-
retry_cfg->retry_http_count = len;
|
|
661
|
-
for (int i = 0; i < len; i++)
|
|
662
|
-
retry_cfg->retry_http_codes[i] = NUM2INT(rb_ary_entry(rc, i));
|
|
663
|
-
}
|
|
664
|
-
}
|
|
1263
|
+
Check_Type(codes, T_ARRAY);
|
|
1264
|
+
len_long = RARRAY_LEN(codes);
|
|
1265
|
+
if (len_long > INT_MAX)
|
|
1266
|
+
rb_raise(rb_eArgError, "retry_codes is too large");
|
|
1267
|
+
|
|
1268
|
+
len = (int)len_long;
|
|
1269
|
+
for (int i = 0; i < len; i++) {
|
|
1270
|
+
int code = NUM2INT(rb_ary_entry(codes, i));
|
|
1271
|
+
if (code < 100 || code > 599)
|
|
1272
|
+
rb_raise(rb_eArgError, "retry_codes must contain valid HTTP status codes");
|
|
665
1273
|
}
|
|
1274
|
+
|
|
1275
|
+
if (len == 0)
|
|
1276
|
+
return;
|
|
1277
|
+
|
|
1278
|
+
retry_cfg->retry_http_codes = malloc(sizeof(int) * (size_t)len);
|
|
1279
|
+
if (!retry_cfg->retry_http_codes)
|
|
1280
|
+
rb_raise(rb_eNoMemError, "failed to allocate retry codes");
|
|
1281
|
+
|
|
1282
|
+
retry_cfg->retry_http_count = len;
|
|
1283
|
+
for (int i = 0; i < len; i++)
|
|
1284
|
+
retry_cfg->retry_http_codes[i] = NUM2INT(rb_ary_entry(codes, i));
|
|
1285
|
+
}
|
|
1286
|
+
|
|
1287
|
+
static void parse_options(VALUE options, request_options_t *opts, int *max_conn,
|
|
1288
|
+
retry_config_t *retry_cfg) {
|
|
1289
|
+
VALUE flag;
|
|
1290
|
+
|
|
1291
|
+
opts->timeout_sec = 30;
|
|
1292
|
+
opts->connect_timeout_ms = DEFAULT_CONNECT_TIMEOUT_MS;
|
|
1293
|
+
opts->total_timeout_ms = 0;
|
|
1294
|
+
opts->follow_redirects = 1;
|
|
1295
|
+
opts->max_redirects = MAX_REDIRECTS;
|
|
1296
|
+
opts->deadline_ms = 0;
|
|
1297
|
+
*max_conn = 20;
|
|
1298
|
+
retry_config_init(retry_cfg);
|
|
1299
|
+
|
|
1300
|
+
if (NIL_P(options))
|
|
1301
|
+
return;
|
|
1302
|
+
|
|
1303
|
+
Check_Type(options, T_HASH);
|
|
1304
|
+
opts->timeout_sec =
|
|
1305
|
+
parse_long_option(options, KEY_TIMEOUT, "timeout", 1, MAX_TIMEOUT, opts->timeout_sec, NULL);
|
|
1306
|
+
opts->connect_timeout_ms =
|
|
1307
|
+
parse_long_option(options, KEY_CONNECT_TIMEOUT, "connect_timeout", 1,
|
|
1308
|
+
MAX_CONNECT_TIMEOUT_MS, opts->connect_timeout_ms, NULL);
|
|
1309
|
+
opts->total_timeout_ms = parse_long_option(options, KEY_TOTAL_TIMEOUT, "total_timeout", 1,
|
|
1310
|
+
MAX_TOTAL_TIMEOUT_MS, opts->total_timeout_ms, NULL);
|
|
1311
|
+
opts->max_redirects = parse_long_option(options, KEY_MAX_REDIRECTS, "max_redirects", 0, 100,
|
|
1312
|
+
opts->max_redirects, NULL);
|
|
1313
|
+
|
|
1314
|
+
flag = hash_aref_key(options, KEY_FOLLOW_REDIRECTS);
|
|
1315
|
+
if (!NIL_P(flag))
|
|
1316
|
+
opts->follow_redirects = RTEST(flag) ? 1L : 0L;
|
|
1317
|
+
|
|
1318
|
+
flag = hash_aref_key(options, KEY_RETRY_NON_IDEMPOTENT);
|
|
1319
|
+
if (!NIL_P(flag))
|
|
1320
|
+
retry_cfg->retry_non_idempotent = RTEST(flag) ? 1 : 0;
|
|
1321
|
+
|
|
1322
|
+
*max_conn = parse_int_option(options, KEY_CONNECTIONS, "connections", 1, MAX_CONNECTIONS,
|
|
1323
|
+
*max_conn, NULL);
|
|
1324
|
+
retry_cfg->max_retries = parse_int_option(options, KEY_RETRIES, "retries", 0, MAX_RETRIES,
|
|
1325
|
+
retry_cfg->max_retries, &retry_cfg->retries_explicit);
|
|
1326
|
+
retry_cfg->retry_delay_ms =
|
|
1327
|
+
parse_long_option(options, KEY_RETRY_DELAY, "retry_delay", 0, MAX_RETRY_DELAY_MS,
|
|
1328
|
+
retry_cfg->retry_delay_ms, NULL);
|
|
1329
|
+
parse_retry_codes(options, retry_cfg);
|
|
1330
|
+
}
|
|
1331
|
+
|
|
1332
|
+
static void multi_session_init(multi_session_t *session, CURLM *multi, int count, int max_conn,
|
|
1333
|
+
long timeout_sec) {
|
|
1334
|
+
memset(session, 0, sizeof(*session));
|
|
1335
|
+
session->multi = multi;
|
|
1336
|
+
session->count = count;
|
|
1337
|
+
session->timeout_ms = timeout_sec * 1000;
|
|
1338
|
+
session->max_connections = max_conn;
|
|
1339
|
+
}
|
|
1340
|
+
|
|
1341
|
+
static void multi_session_configure(CURLM *multi, int max_conn) {
|
|
1342
|
+
curl_multi_setopt(multi, CURLMOPT_MAXCONNECTS, (long)max_conn);
|
|
1343
|
+
curl_multi_setopt(multi, CURLMOPT_MAX_TOTAL_CONNECTIONS, (long)max_conn);
|
|
1344
|
+
#ifdef CURLPIPE_MULTIPLEX
|
|
1345
|
+
curl_multi_setopt(multi, CURLMOPT_PIPELINING, CURLPIPE_MULTIPLEX);
|
|
1346
|
+
#endif
|
|
666
1347
|
}
|
|
667
1348
|
|
|
668
1349
|
typedef struct {
|
|
@@ -673,30 +1354,41 @@ typedef struct {
|
|
|
673
1354
|
|
|
674
1355
|
static VALUE cleanup_session(VALUE arg) {
|
|
675
1356
|
cleanup_ctx_t *ctx = (cleanup_ctx_t *)arg;
|
|
1357
|
+
|
|
676
1358
|
if (ctx->session->requests) {
|
|
677
1359
|
for (int i = 0; i < ctx->session->count; i++) {
|
|
678
|
-
if (ctx->session->requests[i].easy)
|
|
1360
|
+
if (ctx->session->requests[i].easy && ctx->session->requests[i].active)
|
|
679
1361
|
curl_multi_remove_handle(ctx->session->multi, ctx->session->requests[i].easy);
|
|
680
1362
|
request_ctx_free(&ctx->session->requests[i]);
|
|
681
1363
|
}
|
|
682
1364
|
free(ctx->session->requests);
|
|
683
1365
|
ctx->session->requests = NULL;
|
|
684
1366
|
}
|
|
1367
|
+
|
|
685
1368
|
if (ctx->invalid) {
|
|
686
1369
|
free(ctx->invalid);
|
|
687
1370
|
ctx->invalid = NULL;
|
|
688
1371
|
}
|
|
1372
|
+
|
|
689
1373
|
if (ctx->session->multi) {
|
|
690
1374
|
curl_multi_cleanup(ctx->session->multi);
|
|
691
1375
|
ctx->session->multi = NULL;
|
|
692
1376
|
}
|
|
1377
|
+
|
|
693
1378
|
if (ctx->retry_cfg && ctx->retry_cfg->retry_http_codes) {
|
|
694
1379
|
free(ctx->retry_cfg->retry_http_codes);
|
|
695
1380
|
ctx->retry_cfg->retry_http_codes = NULL;
|
|
696
1381
|
}
|
|
1382
|
+
|
|
697
1383
|
return Qnil;
|
|
698
1384
|
}
|
|
699
1385
|
|
|
1386
|
+
static FAST_CURL_NORETURN void cleanup_and_raise(cleanup_ctx_t *cleanup, VALUE exception,
|
|
1387
|
+
const char *message) {
|
|
1388
|
+
cleanup_session((VALUE)cleanup);
|
|
1389
|
+
rb_raise(exception, "%s", message);
|
|
1390
|
+
}
|
|
1391
|
+
|
|
700
1392
|
typedef struct {
|
|
701
1393
|
VALUE requests;
|
|
702
1394
|
VALUE options;
|
|
@@ -705,7 +1397,7 @@ typedef struct {
|
|
|
705
1397
|
multi_session_t *session;
|
|
706
1398
|
int *invalid;
|
|
707
1399
|
retry_config_t *retry_cfg;
|
|
708
|
-
|
|
1400
|
+
request_options_t *opts;
|
|
709
1401
|
} execute_args_t;
|
|
710
1402
|
|
|
711
1403
|
static VALUE internal_execute_body(VALUE arg) {
|
|
@@ -714,114 +1406,196 @@ static VALUE internal_execute_body(VALUE arg) {
|
|
|
714
1406
|
multi_session_t *session = ea->session;
|
|
715
1407
|
int *invalid = ea->invalid;
|
|
716
1408
|
retry_config_t *retry_cfg = ea->retry_cfg;
|
|
717
|
-
|
|
718
|
-
int
|
|
1409
|
+
request_options_t *opts = ea->opts;
|
|
1410
|
+
int deadline_hit = 0;
|
|
1411
|
+
int count = session->count;
|
|
1412
|
+
int target = ea->target;
|
|
1413
|
+
int stream = ea->stream;
|
|
719
1414
|
|
|
720
|
-
int
|
|
721
|
-
for (int i = 0; i < count; i++) {
|
|
722
|
-
VALUE req = rb_ary_entry(requests, i);
|
|
1415
|
+
for (int i = 0; i < count; i++)
|
|
723
1416
|
request_ctx_init(&session->requests[i], i);
|
|
724
|
-
if (!setup_easy_handle(&session->requests[i], req, timeout_sec)) {
|
|
725
|
-
session->requests[i].done = 1;
|
|
726
|
-
invalid[i] = 1;
|
|
727
|
-
continue;
|
|
728
|
-
}
|
|
729
|
-
CURLMcode mc = curl_multi_add_handle(session->multi, session->requests[i].easy);
|
|
730
|
-
if (mc != CURLM_OK) {
|
|
731
|
-
session->requests[i].done = 1;
|
|
732
|
-
invalid[i] = 1;
|
|
733
|
-
continue;
|
|
734
|
-
}
|
|
735
|
-
valid_requests++;
|
|
736
|
-
}
|
|
737
|
-
if (valid_requests == 0)
|
|
738
|
-
session->still_running = 0;
|
|
739
1417
|
|
|
740
1418
|
completion_ctx_t cctx;
|
|
741
|
-
cctx.results = stream ? Qnil : rb_ary_new2(count);
|
|
1419
|
+
cctx.results = stream ? Qnil : ((target > 0) ? rb_ary_new2(target) : rb_ary_new2(count));
|
|
742
1420
|
cctx.completed = 0;
|
|
743
1421
|
cctx.target = target;
|
|
744
1422
|
cctx.stream = stream;
|
|
745
|
-
|
|
1423
|
+
|
|
1424
|
+
if (!stream && target <= 0) {
|
|
746
1425
|
for (int i = 0; i < count; i++)
|
|
747
1426
|
rb_ary_store(cctx.results, i, Qnil);
|
|
748
1427
|
}
|
|
749
1428
|
|
|
750
|
-
|
|
1429
|
+
if (opts->total_timeout_ms > 0)
|
|
1430
|
+
opts->deadline_ms = fast_now_ms() + opts->total_timeout_ms;
|
|
751
1431
|
|
|
752
|
-
|
|
753
|
-
|
|
1432
|
+
run_multi_loop(session, &cctx, requests, invalid, opts, NULL, count);
|
|
1433
|
+
|
|
1434
|
+
if (!stream && target <= 0 && retry_cfg->max_retries > 0) {
|
|
754
1435
|
for (int attempt = 0; attempt < retry_cfg->max_retries; attempt++) {
|
|
755
|
-
|
|
756
|
-
|
|
757
|
-
if (!ri)
|
|
1436
|
+
if (opts->deadline_ms > 0 && fast_now_ms() >= opts->deadline_ms) {
|
|
1437
|
+
deadline_hit = 1;
|
|
758
1438
|
break;
|
|
1439
|
+
}
|
|
1440
|
+
|
|
1441
|
+
int *retry_indices = malloc(sizeof(int) * (size_t)count);
|
|
1442
|
+
if (!retry_indices)
|
|
1443
|
+
rb_raise(rb_eNoMemError, "failed to allocate retry index array");
|
|
1444
|
+
|
|
1445
|
+
int retry_count = 0;
|
|
759
1446
|
for (int i = 0; i < count; i++) {
|
|
760
1447
|
if (invalid[i] || !session->requests[i].done)
|
|
761
1448
|
continue;
|
|
762
1449
|
if (should_retry(&session->requests[i], retry_cfg))
|
|
763
|
-
|
|
1450
|
+
retry_indices[retry_count++] = i;
|
|
764
1451
|
}
|
|
1452
|
+
|
|
765
1453
|
if (retry_count == 0) {
|
|
766
|
-
free(
|
|
1454
|
+
free(retry_indices);
|
|
767
1455
|
break;
|
|
768
1456
|
}
|
|
769
|
-
|
|
770
|
-
|
|
771
|
-
|
|
772
|
-
|
|
773
|
-
|
|
774
|
-
|
|
775
|
-
|
|
776
|
-
|
|
1457
|
+
|
|
1458
|
+
long backoff = retry_backoff_ms(retry_cfg, attempt);
|
|
1459
|
+
if (opts->deadline_ms > 0) {
|
|
1460
|
+
long long left = opts->deadline_ms - fast_now_ms();
|
|
1461
|
+
if (left <= 0) {
|
|
1462
|
+
deadline_hit = 1;
|
|
1463
|
+
free(retry_indices);
|
|
1464
|
+
break;
|
|
1465
|
+
}
|
|
1466
|
+
if (backoff > (long)left)
|
|
1467
|
+
backoff = (long)left;
|
|
777
1468
|
}
|
|
778
|
-
|
|
779
|
-
|
|
1469
|
+
retry_delay_sleep(backoff);
|
|
1470
|
+
|
|
1471
|
+
int runnable_count = 0;
|
|
780
1472
|
for (int r = 0; r < retry_count; r++) {
|
|
781
|
-
int idx =
|
|
1473
|
+
int idx = retry_indices[r];
|
|
782
1474
|
request_ctx_t *rc = &session->requests[idx];
|
|
783
|
-
|
|
1475
|
+
|
|
784
1476
|
if (!request_ctx_reset_for_retry(rc)) {
|
|
785
|
-
rc->done = 1;
|
|
786
1477
|
invalid[idx] = 1;
|
|
787
|
-
continue;
|
|
788
|
-
}
|
|
789
|
-
VALUE req = rb_ary_entry(requests, idx);
|
|
790
|
-
if (!setup_easy_handle(rc, req, timeout_sec)) {
|
|
791
1478
|
rc->done = 1;
|
|
792
|
-
|
|
1479
|
+
rc->setup_error = "Failed to allocate a curl handle for the retry";
|
|
793
1480
|
continue;
|
|
794
1481
|
}
|
|
795
|
-
|
|
796
|
-
|
|
797
|
-
|
|
798
|
-
|
|
799
|
-
|
|
1482
|
+
|
|
1483
|
+
retry_indices[runnable_count++] = idx;
|
|
1484
|
+
}
|
|
1485
|
+
|
|
1486
|
+
if (runnable_count > 0) {
|
|
1487
|
+
cctx.completed = 0;
|
|
1488
|
+
run_multi_loop(session, &cctx, requests, invalid, opts, retry_indices,
|
|
1489
|
+
runnable_count);
|
|
800
1490
|
}
|
|
801
|
-
|
|
802
|
-
|
|
803
|
-
run_multi_loop(session, &cctx);
|
|
1491
|
+
|
|
1492
|
+
free(retry_indices);
|
|
804
1493
|
}
|
|
805
1494
|
}
|
|
806
1495
|
|
|
807
|
-
if (!stream) {
|
|
1496
|
+
if (!stream && target <= 0) {
|
|
808
1497
|
for (int i = 0; i < count; i++) {
|
|
809
1498
|
request_ctx_t *rc = &session->requests[i];
|
|
810
1499
|
VALUE response;
|
|
1500
|
+
|
|
811
1501
|
if (invalid[i]) {
|
|
812
|
-
response = build_error_response(
|
|
1502
|
+
response = build_error_response(rc->setup_error ? rc->setup_error
|
|
1503
|
+
: "Invalid request configuration",
|
|
1504
|
+
ERR_INVALID_REQUEST, rc->attempts);
|
|
1505
|
+
} else if (!rc->done) {
|
|
1506
|
+
response =
|
|
1507
|
+
deadline_hit || (opts->deadline_ms > 0 && fast_now_ms() >= opts->deadline_ms)
|
|
1508
|
+
? build_error_response("Total timeout exceeded", ERR_DEADLINE, rc->attempts)
|
|
1509
|
+
: build_error_response("Request was not completed", ERR_NOT_COMPLETED,
|
|
1510
|
+
rc->attempts);
|
|
813
1511
|
} else if (rc->curl_result == CURLE_OK) {
|
|
814
1512
|
response = build_response(rc);
|
|
815
1513
|
} else {
|
|
816
1514
|
response = build_error_response_with_code(curl_easy_strerror(rc->curl_result),
|
|
817
|
-
(int)rc->curl_result);
|
|
1515
|
+
(int)rc->curl_result, rc->attempts);
|
|
818
1516
|
}
|
|
819
|
-
|
|
1517
|
+
|
|
1518
|
+
rb_ary_store(cctx.results, i, build_result_pair(i, response));
|
|
820
1519
|
}
|
|
821
1520
|
}
|
|
1521
|
+
|
|
822
1522
|
return stream ? Qnil : cctx.results;
|
|
823
1523
|
}
|
|
824
1524
|
|
|
1525
|
+
#ifdef FAST_CURL_HAVE_FIBER_SCHEDULER
|
|
1526
|
+
typedef struct {
|
|
1527
|
+
execute_args_t *ea;
|
|
1528
|
+
VALUE scheduler;
|
|
1529
|
+
VALUE blocker;
|
|
1530
|
+
VALUE fiber;
|
|
1531
|
+
VALUE thread;
|
|
1532
|
+
VALUE result;
|
|
1533
|
+
VALUE exception;
|
|
1534
|
+
int state;
|
|
1535
|
+
int finished;
|
|
1536
|
+
} scheduler_execute_ctx_t;
|
|
1537
|
+
|
|
1538
|
+
static VALUE scheduler_execute_thread(void *arg) {
|
|
1539
|
+
scheduler_execute_ctx_t *ctx = (scheduler_execute_ctx_t *)arg;
|
|
1540
|
+
|
|
1541
|
+
ctx->result = rb_protect(internal_execute_body, (VALUE)ctx->ea, &ctx->state);
|
|
1542
|
+
if (ctx->state) {
|
|
1543
|
+
ctx->exception = rb_errinfo();
|
|
1544
|
+
rb_set_errinfo(Qnil);
|
|
1545
|
+
}
|
|
1546
|
+
|
|
1547
|
+
ctx->finished = 1;
|
|
1548
|
+
rb_fiber_scheduler_unblock(ctx->scheduler, ctx->blocker, ctx->fiber);
|
|
1549
|
+
return Qnil;
|
|
1550
|
+
}
|
|
1551
|
+
|
|
1552
|
+
static void cancel_scheduler_execute(scheduler_execute_ctx_t *ctx) {
|
|
1553
|
+
multi_session_t *session = ctx->ea->session;
|
|
1554
|
+
|
|
1555
|
+
session->cancelled = 1;
|
|
1556
|
+
#ifdef HAVE_CURL_MULTI_WAKEUP
|
|
1557
|
+
if (session->multi)
|
|
1558
|
+
curl_multi_wakeup(session->multi);
|
|
1559
|
+
#endif
|
|
1560
|
+
}
|
|
1561
|
+
|
|
1562
|
+
static VALUE scheduler_execute_wait(VALUE arg) {
|
|
1563
|
+
scheduler_execute_ctx_t *ctx = (scheduler_execute_ctx_t *)arg;
|
|
1564
|
+
|
|
1565
|
+
if (!ctx->finished)
|
|
1566
|
+
rb_fiber_scheduler_block(ctx->scheduler, ctx->blocker, Qnil);
|
|
1567
|
+
|
|
1568
|
+
rb_funcall(ctx->thread, rb_intern("join"), 0);
|
|
1569
|
+
|
|
1570
|
+
if (ctx->state)
|
|
1571
|
+
rb_exc_raise(ctx->exception);
|
|
1572
|
+
|
|
1573
|
+
return ctx->result;
|
|
1574
|
+
}
|
|
1575
|
+
|
|
1576
|
+
static VALUE scheduler_execute_ensure(VALUE arg) {
|
|
1577
|
+
scheduler_execute_ctx_t *ctx = (scheduler_execute_ctx_t *)arg;
|
|
1578
|
+
|
|
1579
|
+
if (!NIL_P(ctx->thread)) {
|
|
1580
|
+
if (!ctx->finished)
|
|
1581
|
+
cancel_scheduler_execute(ctx);
|
|
1582
|
+
rb_funcall(ctx->thread, rb_intern("join"), 0);
|
|
1583
|
+
}
|
|
1584
|
+
|
|
1585
|
+
return Qnil;
|
|
1586
|
+
}
|
|
1587
|
+
|
|
1588
|
+
static VALUE execute_with_fiber_scheduler(VALUE arg) {
|
|
1589
|
+
scheduler_execute_ctx_t *ctx = (scheduler_execute_ctx_t *)arg;
|
|
1590
|
+
|
|
1591
|
+
ctx->blocker = rb_obj_alloc(rb_cObject);
|
|
1592
|
+
ctx->fiber = rb_fiber_current();
|
|
1593
|
+
ctx->thread = rb_thread_create(scheduler_execute_thread, ctx);
|
|
1594
|
+
|
|
1595
|
+
return rb_ensure(scheduler_execute_wait, arg, scheduler_execute_ensure, arg);
|
|
1596
|
+
}
|
|
1597
|
+
#endif
|
|
1598
|
+
|
|
825
1599
|
static VALUE internal_execute(VALUE requests, VALUE options, int target, int stream) {
|
|
826
1600
|
Check_Type(requests, T_ARRAY);
|
|
827
1601
|
|
|
@@ -832,55 +1606,67 @@ static VALUE internal_execute(VALUE requests, VALUE options, int target, int str
|
|
|
832
1606
|
rb_raise(rb_eArgError, "too many requests (%ld), maximum is %d", count_long, MAX_REQUESTS);
|
|
833
1607
|
if (count_long > INT_MAX)
|
|
834
1608
|
rb_raise(rb_eArgError, "request count overflows int");
|
|
1609
|
+
|
|
835
1610
|
int count = (int)count_long;
|
|
836
1611
|
|
|
837
|
-
|
|
1612
|
+
if (target > 0 && target > count)
|
|
1613
|
+
target = count;
|
|
1614
|
+
|
|
1615
|
+
request_options_t opts;
|
|
838
1616
|
int max_conn;
|
|
839
1617
|
retry_config_t retry_cfg;
|
|
840
|
-
parse_options(options, &
|
|
1618
|
+
parse_options(options, &opts, &max_conn, &retry_cfg);
|
|
1619
|
+
|
|
1620
|
+
for (int i = 0; i < count; i++) {
|
|
1621
|
+
request_ctx_t probe;
|
|
1622
|
+
request_ctx_init(&probe, i);
|
|
1623
|
+
if (!request_ctx_prepare_easy(&probe)) {
|
|
1624
|
+
request_ctx_free(&probe);
|
|
1625
|
+
continue;
|
|
1626
|
+
}
|
|
1627
|
+
setup_easy_handle(&probe, rb_ary_entry(requests, i), &opts);
|
|
1628
|
+
int fatal = probe.setup_error_fatal;
|
|
1629
|
+
const char *message = probe.setup_error;
|
|
1630
|
+
char buffer[128];
|
|
1631
|
+
if (fatal && message) {
|
|
1632
|
+
snprintf(buffer, sizeof(buffer), "%s (request %d)", message, i);
|
|
1633
|
+
}
|
|
1634
|
+
request_ctx_free(&probe);
|
|
1635
|
+
if (fatal) {
|
|
1636
|
+
VALUE klass = NIL_P(fast_validation_error) ? rb_eArgError : fast_validation_error;
|
|
1637
|
+
rb_raise(klass, "%s", buffer);
|
|
1638
|
+
}
|
|
1639
|
+
}
|
|
841
1640
|
|
|
842
1641
|
if (stream || target > 0) {
|
|
843
|
-
if (retry_cfg.max_retries > 0 && stream)
|
|
844
|
-
rb_warn(
|
|
845
|
-
|
|
846
|
-
if (retry_cfg.max_retries > 0 && target > 0)
|
|
847
|
-
rb_warn(
|
|
848
|
-
|
|
1642
|
+
if (retry_cfg.retries_explicit && retry_cfg.max_retries > 0 && stream)
|
|
1643
|
+
rb_warn(
|
|
1644
|
+
"FastCurl: retries are not supported in stream_execute, ignoring retries option");
|
|
1645
|
+
if (retry_cfg.retries_explicit && retry_cfg.max_retries > 0 && target > 0)
|
|
1646
|
+
rb_warn(
|
|
1647
|
+
"FastCurl: retries are not supported in first_execute, ignoring retries option");
|
|
849
1648
|
retry_cfg.max_retries = 0;
|
|
850
1649
|
}
|
|
851
1650
|
|
|
852
1651
|
multi_session_t session;
|
|
853
|
-
|
|
854
|
-
session
|
|
855
|
-
session.timeout_ms = timeout_sec * 1000;
|
|
856
|
-
session.max_connections = max_conn;
|
|
857
|
-
session.cancelled = 0;
|
|
858
|
-
session.requests = NULL;
|
|
859
|
-
|
|
860
|
-
curl_multi_setopt(session.multi, CURLMOPT_MAXCONNECTS, (long)max_conn);
|
|
861
|
-
curl_multi_setopt(session.multi, CURLMOPT_MAX_TOTAL_CONNECTIONS, (long)max_conn);
|
|
862
|
-
#ifdef CURLPIPE_MULTIPLEX
|
|
863
|
-
curl_multi_setopt(session.multi, CURLMOPT_PIPELINING, CURLPIPE_MULTIPLEX);
|
|
864
|
-
#endif
|
|
1652
|
+
int *invalid = NULL;
|
|
1653
|
+
multi_session_init(&session, curl_multi_init(), count, max_conn, opts.timeout_sec);
|
|
865
1654
|
|
|
866
|
-
session.
|
|
867
|
-
if (!session.requests) {
|
|
868
|
-
curl_multi_cleanup(session.multi);
|
|
869
|
-
if (retry_cfg.retry_http_codes)
|
|
870
|
-
free(retry_cfg.retry_http_codes);
|
|
871
|
-
rb_raise(rb_eNoMemError, "failed to allocate request contexts");
|
|
872
|
-
}
|
|
1655
|
+
cleanup_ctx_t cleanup = {.session = &session, .invalid = NULL, .retry_cfg = &retry_cfg};
|
|
873
1656
|
|
|
874
|
-
|
|
875
|
-
|
|
876
|
-
|
|
877
|
-
|
|
878
|
-
if (retry_cfg.retry_http_codes)
|
|
879
|
-
free(retry_cfg.retry_http_codes);
|
|
880
|
-
rb_raise(rb_eNoMemError, "failed to allocate tracking array");
|
|
881
|
-
}
|
|
1657
|
+
if (!session.multi)
|
|
1658
|
+
cleanup_and_raise(&cleanup, rb_eNoMemError, "failed to initialize curl multi handle");
|
|
1659
|
+
|
|
1660
|
+
multi_session_configure(session.multi, max_conn);
|
|
882
1661
|
|
|
883
|
-
|
|
1662
|
+
session.requests = calloc((size_t)count, sizeof(request_ctx_t));
|
|
1663
|
+
if (!session.requests)
|
|
1664
|
+
cleanup_and_raise(&cleanup, rb_eNoMemError, "failed to allocate request contexts");
|
|
1665
|
+
|
|
1666
|
+
invalid = calloc((size_t)count, sizeof(int));
|
|
1667
|
+
cleanup.invalid = invalid;
|
|
1668
|
+
if (!invalid)
|
|
1669
|
+
cleanup_and_raise(&cleanup, rb_eNoMemError, "failed to allocate tracking array");
|
|
884
1670
|
execute_args_t ea = {
|
|
885
1671
|
.requests = requests,
|
|
886
1672
|
.options = options,
|
|
@@ -889,8 +1675,28 @@ static VALUE internal_execute(VALUE requests, VALUE options, int target, int str
|
|
|
889
1675
|
.session = &session,
|
|
890
1676
|
.invalid = invalid,
|
|
891
1677
|
.retry_cfg = &retry_cfg,
|
|
892
|
-
.
|
|
1678
|
+
.opts = &opts,
|
|
893
1679
|
};
|
|
1680
|
+
|
|
1681
|
+
#ifdef FAST_CURL_HAVE_FIBER_SCHEDULER
|
|
1682
|
+
VALUE scheduler = current_fiber_scheduler();
|
|
1683
|
+
if (scheduler != Qnil && !stream) {
|
|
1684
|
+
scheduler_execute_ctx_t scheduler_ctx = {
|
|
1685
|
+
.ea = &ea,
|
|
1686
|
+
.scheduler = scheduler,
|
|
1687
|
+
.blocker = Qnil,
|
|
1688
|
+
.fiber = Qnil,
|
|
1689
|
+
.thread = Qnil,
|
|
1690
|
+
.result = Qnil,
|
|
1691
|
+
.exception = Qnil,
|
|
1692
|
+
.state = 0,
|
|
1693
|
+
.finished = 0,
|
|
1694
|
+
};
|
|
1695
|
+
|
|
1696
|
+
return rb_ensure(execute_with_fiber_scheduler, (VALUE)&scheduler_ctx, cleanup_session,
|
|
1697
|
+
(VALUE)&cleanup);
|
|
1698
|
+
}
|
|
1699
|
+
#endif
|
|
894
1700
|
return rb_ensure(internal_execute_body, (VALUE)&ea, cleanup_session, (VALUE)&cleanup);
|
|
895
1701
|
}
|
|
896
1702
|
|
|
@@ -903,67 +1709,53 @@ static VALUE rb_fast_curl_execute(int argc, VALUE *argv, VALUE self) {
|
|
|
903
1709
|
static VALUE rb_fast_curl_first_execute(int argc, VALUE *argv, VALUE self) {
|
|
904
1710
|
VALUE requests, options;
|
|
905
1711
|
rb_scan_args(argc, argv, "1:", &requests, &options);
|
|
1712
|
+
|
|
906
1713
|
int count = 1;
|
|
907
1714
|
if (!NIL_P(options)) {
|
|
908
|
-
|
|
1715
|
+
Check_Type(options, T_HASH);
|
|
1716
|
+
VALUE c = hash_aref_key(options, KEY_COUNT_OPT);
|
|
909
1717
|
if (!NIL_P(c))
|
|
910
1718
|
count = NUM2INT(c);
|
|
911
1719
|
}
|
|
1720
|
+
|
|
1721
|
+
if (count <= 0)
|
|
1722
|
+
rb_raise(rb_eArgError, "count must be positive");
|
|
1723
|
+
|
|
912
1724
|
return internal_execute(requests, options, count, 0);
|
|
913
1725
|
}
|
|
914
1726
|
|
|
915
1727
|
static VALUE rb_fast_curl_stream_execute(int argc, VALUE *argv, VALUE self) {
|
|
916
1728
|
VALUE requests, options;
|
|
917
1729
|
rb_scan_args(argc, argv, "1:", &requests, &options);
|
|
1730
|
+
|
|
918
1731
|
if (!rb_block_given_p())
|
|
919
1732
|
rb_raise(rb_eArgError, "stream_execute requires a block");
|
|
1733
|
+
|
|
920
1734
|
return internal_execute(requests, options, -1, 1);
|
|
921
1735
|
}
|
|
922
1736
|
|
|
923
1737
|
void Init_fast_curl(void) {
|
|
924
1738
|
curl_global_init(CURL_GLOBAL_ALL);
|
|
1739
|
+
fast_share_init();
|
|
1740
|
+
|
|
1741
|
+
for (int i = 0; i < KEY_LAST; i++) {
|
|
1742
|
+
fast_ids[i] = rb_intern(KEY_NAMES[i]);
|
|
1743
|
+
fast_syms[i] = ID2SYM(fast_ids[i]);
|
|
1744
|
+
rb_gc_register_address(&fast_syms[i]);
|
|
1745
|
+
}
|
|
925
1746
|
|
|
926
|
-
|
|
927
|
-
|
|
928
|
-
|
|
929
|
-
|
|
930
|
-
id_url = rb_intern("url");
|
|
931
|
-
id_method = rb_intern("method");
|
|
932
|
-
id_timeout = rb_intern("timeout");
|
|
933
|
-
id_connections = rb_intern("connections");
|
|
934
|
-
id_count = rb_intern("count");
|
|
935
|
-
id_keys = rb_intern("keys");
|
|
936
|
-
id_retries = rb_intern("retries");
|
|
937
|
-
id_retry_delay = rb_intern("retry_delay");
|
|
938
|
-
id_retry_codes = rb_intern("retry_codes");
|
|
939
|
-
|
|
940
|
-
sym_status = ID2SYM(id_status);
|
|
941
|
-
rb_gc_register_address(&sym_status);
|
|
942
|
-
sym_headers = ID2SYM(id_headers);
|
|
943
|
-
rb_gc_register_address(&sym_headers);
|
|
944
|
-
sym_body = ID2SYM(id_body);
|
|
945
|
-
rb_gc_register_address(&sym_body);
|
|
946
|
-
sym_error_code = ID2SYM(id_error_code);
|
|
947
|
-
rb_gc_register_address(&sym_error_code);
|
|
948
|
-
sym_url = ID2SYM(id_url);
|
|
949
|
-
rb_gc_register_address(&sym_url);
|
|
950
|
-
sym_method = ID2SYM(id_method);
|
|
951
|
-
rb_gc_register_address(&sym_method);
|
|
952
|
-
sym_timeout = ID2SYM(id_timeout);
|
|
953
|
-
rb_gc_register_address(&sym_timeout);
|
|
954
|
-
sym_connections = ID2SYM(id_connections);
|
|
955
|
-
rb_gc_register_address(&sym_connections);
|
|
956
|
-
sym_count = ID2SYM(id_count);
|
|
957
|
-
rb_gc_register_address(&sym_count);
|
|
958
|
-
sym_retries = ID2SYM(id_retries);
|
|
959
|
-
rb_gc_register_address(&sym_retries);
|
|
960
|
-
sym_retry_delay = ID2SYM(id_retry_delay);
|
|
961
|
-
rb_gc_register_address(&sym_retry_delay);
|
|
962
|
-
sym_retry_codes = ID2SYM(id_retry_codes);
|
|
963
|
-
rb_gc_register_address(&sym_retry_codes);
|
|
1747
|
+
for (int i = 0; i < ERR_LAST; i++) {
|
|
1748
|
+
error_syms[i] = ID2SYM(rb_intern(ERROR_KIND_NAMES[i]));
|
|
1749
|
+
rb_gc_register_address(&error_syms[i]);
|
|
1750
|
+
}
|
|
964
1751
|
|
|
965
1752
|
VALUE mFastCurl = rb_define_module("FastCurl");
|
|
966
1753
|
|
|
1754
|
+
if (rb_const_defined_at(mFastCurl, rb_intern("Headers"))) {
|
|
1755
|
+
fast_cHeaders = rb_const_get_at(mFastCurl, rb_intern("Headers"));
|
|
1756
|
+
rb_gc_register_address(&fast_cHeaders);
|
|
1757
|
+
}
|
|
1758
|
+
|
|
967
1759
|
rb_define_module_function(mFastCurl, "execute", rb_fast_curl_execute, -1);
|
|
968
1760
|
rb_define_module_function(mFastCurl, "first_execute", rb_fast_curl_first_execute, -1);
|
|
969
1761
|
rb_define_module_function(mFastCurl, "stream_execute", rb_fast_curl_stream_execute, -1);
|