fast_curl 0.2.0 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +49 -12
- data/ext/fast_curl/extconf.rb +1 -0
- data/ext/fast_curl/fast_curl.c +336 -363
- data/lib/fast_curl/version.rb +1 -1
- metadata +3 -3
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: 00f4a847ee360180fa617212693206970a8e7af85ef02e6370d37cb62d2b384e
|
|
4
|
+
data.tar.gz: f5633dc59f41b4b31efa6e07eb32bc6a85b146e85126fa68b24b742294df5e8b
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: b7f2e299c89da40d0a546e8545310430a46f0cca8abafff53e46953871c4c205d8fc53c16e33e65b9ff3a22c951d212f41f562476977200de84136ce0342cacc
|
|
7
|
+
data.tar.gz: a5fa06d7e44c0846de3cccd3c955a136fd7ee05e01611f05b940bf174cbd356315d23418ebc40ea77d94667c928ae3f2c7087c227df0ea7a39c0b3a27f52da77
|
data/README.md
CHANGED
|
@@ -12,7 +12,9 @@ Ultra-fast parallel HTTP client for Ruby. C extension built on libcurl `curl_mul
|
|
|
12
12
|
|
|
13
13
|
## Installation
|
|
14
14
|
|
|
15
|
-
**Requirements**: Ruby >= 3.
|
|
15
|
+
**Requirements**: Ruby >= 3.1, libcurl
|
|
16
|
+
|
|
17
|
+
> **Why Ruby 3.1?** The C extension uses `rb_fiber_scheduler_current`, `rb_fiber_scheduler_block` and `rb_fiber_scheduler_unblock` to properly yield control to the Fiber Scheduler during I/O. These APIs are stable starting from Ruby 3.1. Without them, there is no correct way for a C extension to cooperate with the scheduler — earlier approaches (`rb_thread_schedule`) hold the GVL and block other fibers.
|
|
16
18
|
|
|
17
19
|
```ruby
|
|
18
20
|
gem 'fast_curl'
|
|
@@ -142,19 +144,54 @@ end
|
|
|
142
144
|
|
|
143
145
|
## Performance
|
|
144
146
|
|
|
145
|
-
|
|
147
|
+
Benchmarks against `httpbin.org`, 5 iterations with 1 warmup, median times.
|
|
148
|
+
Run yourself: `bundle exec ruby benchmark/local_bench.rb`.
|
|
149
|
+
|
|
150
|
+
Each request hits `/delay/1` (server-side 1-second delay), so sequential baseline
|
|
151
|
+
grows linearly while parallel clients stay near ~1s plus network overhead.
|
|
152
|
+
|
|
153
|
+
### Time to completion (lower is better)
|
|
154
|
+
|
|
155
|
+
| Scenario | Net::HTTP sequential | fast_curl (thread) | fast_curl (fiber/Async) | Async::HTTP::Internet |
|
|
156
|
+
|---------------------------------|---------------------:|-------------------:|------------------------:|----------------------:|
|
|
157
|
+
| 4 requests × 1s, conn=4 | 8.27s | 2.36s | 2.13s | 2.56s |
|
|
158
|
+
| 10 requests × 1s, conn=10 | 20.92s | 3.49s | 5.23s | 3.83s |
|
|
159
|
+
| 20 requests × 1s, conn=5 | 42.56s | 2.94s | 2.90s | 12.14s |
|
|
160
|
+
| 200 requests × 1s, conn=20 | — | 22.19s | 21.77s | 23.59s |
|
|
161
|
+
|
|
162
|
+
### Speedup vs Net::HTTP (median)
|
|
163
|
+
|
|
164
|
+
| Scenario | fast_curl (thread) | fast_curl (fiber) | Async::HTTP |
|
|
165
|
+
|-----------------------------------|-------------------:|------------------:|------------:|
|
|
166
|
+
| 4 requests × 1s | **3.5x** | 3.9x | 3.2x |
|
|
167
|
+
| 10 requests × 1s | **6.0x** | 4.0x | 5.5x |
|
|
168
|
+
| 20 requests × 1s (queued, conn=5) | **14.5x** | 14.7x | 3.5x |
|
|
169
|
+
|
|
170
|
+
### Memory & allocations per request batch (lower is better)
|
|
171
|
+
|
|
172
|
+
| Scenario | fast_curl (thread) allocated | fast_curl (fiber) allocated | Async::HTTP allocated |
|
|
173
|
+
|---------------------------------|-----------------------------:|----------------------------:|----------------------:|
|
|
174
|
+
| 4 requests × 1s | **278 obj** | 350 obj | 2,433 obj |
|
|
175
|
+
| 10 requests × 1s | **490 obj** | 756 obj | 4,763 obj |
|
|
176
|
+
| 20 requests × 1s, conn=5 | **621 obj** | 750 obj | 8,536 obj |
|
|
177
|
+
| 200 requests × 1s, conn=20 | **5,188 obj** | 5,642 obj | 78,203 obj |
|
|
178
|
+
|
|
179
|
+
Ruby heap delta stays near zero across all scenarios for fast_curl — most allocation
|
|
180
|
+
happens in C, not on the Ruby heap.
|
|
181
|
+
|
|
182
|
+
### Error handling
|
|
183
|
+
|
|
184
|
+
| Scenario | Time |
|
|
185
|
+
|--------------------------------------------------------------|------:|
|
|
186
|
+
| 4 mixed requests (404, 500, DNS fail, 30s delay), timeout=2s | 4.02s |
|
|
187
|
+
|
|
188
|
+
Bounded by `timeout=2s` rather than by the slow request.
|
|
146
189
|
|
|
147
|
-
|
|
148
|
-
|--------|------------|-------------|--------------|---------------|
|
|
149
|
-
| Net::HTTP sequential | 7.93s (+2.1 MB) | 24.20s (+0.3 MB) | 48.58s (+1.2 MB) | - |
|
|
150
|
-
| fast_curl (thread) | 2.09s (+0.7 MB) | 3.73s (+0.9 MB) | 3.76s (+0.0 MB) | 5.88s (+2.3 MB) |
|
|
151
|
-
| fast_curl (fiber) | 1.96s (+0.4 MB) | 4.86s (+0.0 MB) | 3.71s (+0.2 MB) | 9.60s (+1.6 MB) |
|
|
152
|
-
| Async::HTTP | 2.54s (+0.3 MB) | 4.27s (+0.4 MB) | 9.16s (+0.5 MB) | 22.44s (+10.7 MB) |
|
|
190
|
+
### Notes on the numbers
|
|
153
191
|
|
|
154
|
-
|
|
155
|
-
-
|
|
156
|
-
-
|
|
157
|
-
- Error handling (timeout=2s): 2.01s (+0.0 MB)
|
|
192
|
+
- **Net::HTTP sequential** is the proof-of-parallelism baseline — it confirms fast_curl and Async are actually running concurrently, not that they "beat" a different library. 4×1s sequentially = 4s, parallel = ~1s + overhead.
|
|
193
|
+
- **Variance is high against remote endpoints** (httpbin.org). For stable numbers, use `--local` which spawns a WEBrick server on 127.0.0.1.
|
|
194
|
+
- **fast_curl (thread) vs (fiber)**: same underlying C code, different scheduling. "thread" is the default; "fiber" kicks in automatically when called inside `Async do ... end`.
|
|
158
195
|
|
|
159
196
|
## License
|
|
160
197
|
|
data/ext/fast_curl/extconf.rb
CHANGED
|
@@ -6,6 +6,7 @@ abort "curl/curl.h is required" unless have_header("curl/curl.h")
|
|
|
6
6
|
have_header("ruby/thread.h")
|
|
7
7
|
have_header("ruby/fiber/scheduler.h")
|
|
8
8
|
|
|
9
|
+
have_func("curl_multi_wakeup", "curl/curl.h")
|
|
9
10
|
have_func("rb_fiber_scheduler_current", "ruby.h")
|
|
10
11
|
have_func("rb_io_wait", "ruby.h")
|
|
11
12
|
|
data/ext/fast_curl/fast_curl.c
CHANGED
|
@@ -12,6 +12,7 @@
|
|
|
12
12
|
#define MAX_REDIRECTS 5
|
|
13
13
|
#define MAX_TIMEOUT 300
|
|
14
14
|
#define MAX_RETRIES 10
|
|
15
|
+
#define MAX_REQUESTS 10000
|
|
15
16
|
#define DEFAULT_RETRIES 1
|
|
16
17
|
#define DEFAULT_RETRY_DELAY 0
|
|
17
18
|
#define INITIAL_BUF_CAP 8192
|
|
@@ -26,31 +27,12 @@ static const CURLcode DEFAULT_RETRYABLE_CURLE[] = {
|
|
|
26
27
|
#define DEFAULT_RETRYABLE_CURLE_COUNT \
|
|
27
28
|
(int)(sizeof(DEFAULT_RETRYABLE_CURLE) / sizeof(DEFAULT_RETRYABLE_CURLE[0]))
|
|
28
29
|
|
|
29
|
-
static ID id_status;
|
|
30
|
-
static ID
|
|
31
|
-
static ID
|
|
32
|
-
static
|
|
33
|
-
static
|
|
34
|
-
static
|
|
35
|
-
static ID id_timeout;
|
|
36
|
-
static ID id_connections;
|
|
37
|
-
static ID id_count;
|
|
38
|
-
static ID id_keys;
|
|
39
|
-
static ID id_retries;
|
|
40
|
-
static ID id_retry_delay;
|
|
41
|
-
static ID id_retry_codes;
|
|
42
|
-
static VALUE sym_status;
|
|
43
|
-
static VALUE sym_headers;
|
|
44
|
-
static VALUE sym_body;
|
|
45
|
-
static VALUE sym_error_code;
|
|
46
|
-
static VALUE sym_url;
|
|
47
|
-
static VALUE sym_method;
|
|
48
|
-
static VALUE sym_timeout;
|
|
49
|
-
static VALUE sym_connections;
|
|
50
|
-
static VALUE sym_count;
|
|
51
|
-
static VALUE sym_retries;
|
|
52
|
-
static VALUE sym_retry_delay;
|
|
53
|
-
static VALUE sym_retry_codes;
|
|
30
|
+
static ID id_status, id_headers, id_body, id_error_code, id_url, id_method;
|
|
31
|
+
static ID id_timeout, id_connections, id_count, id_keys;
|
|
32
|
+
static ID id_retries, id_retry_delay, id_retry_codes;
|
|
33
|
+
static VALUE sym_status, sym_headers, sym_body, sym_error_code, sym_url, sym_method;
|
|
34
|
+
static VALUE sym_timeout, sym_connections, sym_count;
|
|
35
|
+
static VALUE sym_retries, sym_retry_delay, sym_retry_codes;
|
|
54
36
|
|
|
55
37
|
typedef struct {
|
|
56
38
|
char *data;
|
|
@@ -82,26 +64,20 @@ static inline void buffer_reset(buffer_t *buf) {
|
|
|
82
64
|
static size_t write_callback(char *ptr, size_t size, size_t nmemb, void *userdata) {
|
|
83
65
|
buffer_t *buf = (buffer_t *)userdata;
|
|
84
66
|
size_t total = size * nmemb;
|
|
85
|
-
|
|
86
|
-
if (buf->len + total > buf->max_size) {
|
|
67
|
+
if (buf->len + total > buf->max_size)
|
|
87
68
|
return 0;
|
|
88
|
-
}
|
|
89
|
-
|
|
90
69
|
if (buf->len + total >= buf->cap) {
|
|
91
70
|
size_t new_cap = (buf->cap == 0) ? INITIAL_BUF_CAP : buf->cap;
|
|
92
71
|
while (new_cap <= buf->len + total)
|
|
93
72
|
new_cap *= 2;
|
|
94
|
-
|
|
95
73
|
if (new_cap > buf->max_size)
|
|
96
74
|
new_cap = buf->max_size;
|
|
97
|
-
|
|
98
75
|
char *new_data = realloc(buf->data, new_cap);
|
|
99
76
|
if (!new_data)
|
|
100
77
|
return 0;
|
|
101
78
|
buf->data = new_data;
|
|
102
79
|
buf->cap = new_cap;
|
|
103
80
|
}
|
|
104
|
-
|
|
105
81
|
memcpy(buf->data + buf->len, ptr, total);
|
|
106
82
|
buf->len += total;
|
|
107
83
|
return total;
|
|
@@ -142,33 +118,27 @@ static void header_list_reset(header_list_t *h) {
|
|
|
142
118
|
static size_t header_callback(char *ptr, size_t size, size_t nmemb, void *userdata) {
|
|
143
119
|
header_list_t *h = (header_list_t *)userdata;
|
|
144
120
|
size_t total = size * nmemb;
|
|
145
|
-
|
|
146
121
|
if (total <= 2)
|
|
147
122
|
return total;
|
|
148
|
-
|
|
149
123
|
if (h->count >= h->cap) {
|
|
150
124
|
int new_cap = (h->cap == 0) ? INITIAL_HEADER_CAP : h->cap * 2;
|
|
151
|
-
header_entry_t *
|
|
152
|
-
if (!
|
|
125
|
+
header_entry_t *ne = realloc(h->entries, sizeof(header_entry_t) * new_cap);
|
|
126
|
+
if (!ne)
|
|
153
127
|
return 0;
|
|
154
|
-
h->entries =
|
|
128
|
+
h->entries = ne;
|
|
155
129
|
h->cap = new_cap;
|
|
156
130
|
}
|
|
157
|
-
|
|
158
131
|
size_t stripped = total;
|
|
159
132
|
while (stripped > 0 && (ptr[stripped - 1] == '\r' || ptr[stripped - 1] == '\n'))
|
|
160
133
|
stripped--;
|
|
161
|
-
|
|
162
134
|
char *entry = malloc(stripped + 1);
|
|
163
135
|
if (!entry)
|
|
164
136
|
return 0;
|
|
165
137
|
memcpy(entry, ptr, stripped);
|
|
166
138
|
entry[stripped] = '\0';
|
|
167
|
-
|
|
168
139
|
h->entries[h->count].str = entry;
|
|
169
140
|
h->entries[h->count].len = stripped;
|
|
170
141
|
h->count++;
|
|
171
|
-
|
|
172
142
|
return size * nmemb;
|
|
173
143
|
}
|
|
174
144
|
|
|
@@ -234,6 +204,7 @@ typedef struct {
|
|
|
234
204
|
int still_running;
|
|
235
205
|
long timeout_ms;
|
|
236
206
|
int max_connections;
|
|
207
|
+
volatile int cancelled;
|
|
237
208
|
} multi_session_t;
|
|
238
209
|
|
|
239
210
|
typedef struct {
|
|
@@ -243,85 +214,124 @@ typedef struct {
|
|
|
243
214
|
int retry_http_count;
|
|
244
215
|
} retry_config_t;
|
|
245
216
|
|
|
217
|
+
static int contains_header_injection(const char *str, long len) {
|
|
218
|
+
for (long i = 0; i < len; i++) {
|
|
219
|
+
if (str[i] == '\r' || str[i] == '\n' || str[i] == '\0')
|
|
220
|
+
return 1;
|
|
221
|
+
}
|
|
222
|
+
return 0;
|
|
223
|
+
}
|
|
224
|
+
|
|
225
|
+
#ifdef HAVE_RB_FIBER_SCHEDULER_CURRENT
|
|
226
|
+
static VALUE current_fiber_scheduler(void) {
|
|
227
|
+
VALUE sched = rb_fiber_scheduler_current();
|
|
228
|
+
if (sched == Qnil || sched == Qfalse)
|
|
229
|
+
return Qnil;
|
|
230
|
+
return sched;
|
|
231
|
+
}
|
|
232
|
+
#else
|
|
233
|
+
static VALUE current_fiber_scheduler(void) {
|
|
234
|
+
return Qnil;
|
|
235
|
+
}
|
|
236
|
+
#endif
|
|
237
|
+
|
|
238
|
+
typedef struct {
|
|
239
|
+
void *(*func)(void *);
|
|
240
|
+
void *arg;
|
|
241
|
+
VALUE scheduler;
|
|
242
|
+
VALUE blocker;
|
|
243
|
+
VALUE fiber;
|
|
244
|
+
} fiber_worker_ctx_t;
|
|
245
|
+
|
|
246
|
+
static void *fiber_worker_nogvl(void *arg) {
|
|
247
|
+
fiber_worker_ctx_t *c = (fiber_worker_ctx_t *)arg;
|
|
248
|
+
c->func(c->arg);
|
|
249
|
+
return NULL;
|
|
250
|
+
}
|
|
251
|
+
|
|
252
|
+
static VALUE fiber_worker_thread(void *arg) {
|
|
253
|
+
fiber_worker_ctx_t *c = (fiber_worker_ctx_t *)arg;
|
|
254
|
+
rb_thread_call_without_gvl(fiber_worker_nogvl, c, RUBY_UBF_PROCESS, NULL);
|
|
255
|
+
rb_fiber_scheduler_unblock(c->scheduler, c->blocker, c->fiber);
|
|
256
|
+
return Qnil;
|
|
257
|
+
}
|
|
258
|
+
|
|
259
|
+
static void run_via_fiber_worker(VALUE scheduler, void *(*func)(void *), void *arg) {
|
|
260
|
+
fiber_worker_ctx_t ctx = {
|
|
261
|
+
.func = func,
|
|
262
|
+
.arg = arg,
|
|
263
|
+
.scheduler = scheduler,
|
|
264
|
+
.blocker = rb_obj_alloc(rb_cObject),
|
|
265
|
+
.fiber = rb_fiber_current(),
|
|
266
|
+
};
|
|
267
|
+
VALUE th = rb_thread_create(fiber_worker_thread, &ctx);
|
|
268
|
+
rb_fiber_scheduler_block(scheduler, ctx.blocker, Qnil);
|
|
269
|
+
rb_funcall(th, rb_intern("join"), 0);
|
|
270
|
+
}
|
|
271
|
+
|
|
246
272
|
static VALUE build_response(request_ctx_t *ctx) {
|
|
247
273
|
long status = 0;
|
|
248
274
|
curl_easy_getinfo(ctx->easy, CURLINFO_RESPONSE_CODE, &status);
|
|
249
|
-
|
|
250
275
|
VALUE headers_hash = rb_hash_new();
|
|
251
276
|
for (int i = 0; i < ctx->headers.count; i++) {
|
|
252
277
|
const char *hdr = ctx->headers.entries[i].str;
|
|
253
278
|
size_t hdr_len = ctx->headers.entries[i].len;
|
|
254
|
-
|
|
255
279
|
const char *colon = memchr(hdr, ':', hdr_len);
|
|
256
280
|
if (!colon)
|
|
257
281
|
continue;
|
|
258
|
-
|
|
259
282
|
VALUE key = rb_str_new(hdr, colon - hdr);
|
|
260
|
-
|
|
261
|
-
|
|
262
|
-
|
|
263
|
-
|
|
264
|
-
|
|
265
|
-
|
|
266
|
-
|
|
267
|
-
while (val_end > val_start && (*(val_end - 1) == ' ' || *(val_end - 1) == '\t'))
|
|
268
|
-
val_end--;
|
|
269
|
-
|
|
270
|
-
VALUE val = rb_str_new(val_start, val_end - val_start);
|
|
283
|
+
const char *vs = colon + 1, *ve = hdr + hdr_len;
|
|
284
|
+
while (vs < ve && (*vs == ' ' || *vs == '\t'))
|
|
285
|
+
vs++;
|
|
286
|
+
while (ve > vs && (*(ve - 1) == ' ' || *(ve - 1) == '\t'))
|
|
287
|
+
ve--;
|
|
288
|
+
VALUE val = rb_str_new(vs, ve - vs);
|
|
271
289
|
rb_hash_aset(headers_hash, key, val);
|
|
272
290
|
}
|
|
273
|
-
|
|
274
291
|
VALUE body_str =
|
|
275
292
|
ctx->body.data ? rb_str_new(ctx->body.data, ctx->body.len) : rb_str_new_cstr("");
|
|
276
|
-
|
|
277
293
|
VALUE result = rb_hash_new();
|
|
278
294
|
rb_hash_aset(result, sym_status, LONG2NUM(status));
|
|
279
295
|
rb_hash_aset(result, sym_headers, headers_hash);
|
|
280
296
|
rb_hash_aset(result, sym_body, body_str);
|
|
281
|
-
|
|
282
297
|
return result;
|
|
283
298
|
}
|
|
284
299
|
|
|
285
300
|
static VALUE build_error_response(const char *message) {
|
|
286
|
-
VALUE
|
|
287
|
-
rb_hash_aset(
|
|
288
|
-
rb_hash_aset(
|
|
289
|
-
rb_hash_aset(
|
|
290
|
-
return
|
|
301
|
+
VALUE r = rb_hash_new();
|
|
302
|
+
rb_hash_aset(r, sym_status, INT2NUM(0));
|
|
303
|
+
rb_hash_aset(r, sym_headers, Qnil);
|
|
304
|
+
rb_hash_aset(r, sym_body, rb_str_new_cstr(message));
|
|
305
|
+
return r;
|
|
291
306
|
}
|
|
292
307
|
|
|
293
308
|
static VALUE build_error_response_with_code(const char *message, int error_code) {
|
|
294
|
-
VALUE
|
|
295
|
-
rb_hash_aset(
|
|
296
|
-
rb_hash_aset(
|
|
297
|
-
rb_hash_aset(
|
|
298
|
-
rb_hash_aset(
|
|
299
|
-
return
|
|
309
|
+
VALUE r = rb_hash_new();
|
|
310
|
+
rb_hash_aset(r, sym_status, INT2NUM(0));
|
|
311
|
+
rb_hash_aset(r, sym_headers, Qnil);
|
|
312
|
+
rb_hash_aset(r, sym_body, rb_str_new_cstr(message));
|
|
313
|
+
rb_hash_aset(r, sym_error_code, INT2NUM(error_code));
|
|
314
|
+
return r;
|
|
300
315
|
}
|
|
301
316
|
|
|
302
317
|
static int is_valid_url(const char *url) {
|
|
303
318
|
if (!url)
|
|
304
319
|
return 0;
|
|
305
|
-
|
|
306
|
-
|
|
307
|
-
|
|
308
|
-
if (url_len < 8 || url_len > 2048)
|
|
320
|
+
size_t len = strlen(url);
|
|
321
|
+
if (len < 8 || len > 2048)
|
|
309
322
|
return 0;
|
|
310
|
-
|
|
311
323
|
if (strncmp(url, "https://", 8) == 0)
|
|
312
324
|
return 1;
|
|
313
|
-
if (
|
|
325
|
+
if (len >= 7 && strncmp(url, "http://", 7) == 0)
|
|
314
326
|
return 1;
|
|
315
|
-
|
|
316
327
|
return 0;
|
|
317
328
|
}
|
|
318
329
|
|
|
319
|
-
#define CURL_SETOPT_CHECK(handle, option, value)
|
|
320
|
-
do {
|
|
321
|
-
CURLcode
|
|
322
|
-
if (
|
|
323
|
-
return
|
|
324
|
-
} \
|
|
330
|
+
#define CURL_SETOPT_CHECK(handle, option, value) \
|
|
331
|
+
do { \
|
|
332
|
+
CURLcode _r = curl_easy_setopt(handle, option, value); \
|
|
333
|
+
if (_r != CURLE_OK) \
|
|
334
|
+
return _r; \
|
|
325
335
|
} while (0)
|
|
326
336
|
|
|
327
337
|
static CURLcode setup_basic_options(CURL *easy, const char *url_str, long timeout_sec,
|
|
@@ -338,14 +348,12 @@ static CURLcode setup_basic_options(CURL *easy, const char *url_str, long timeou
|
|
|
338
348
|
CURL_SETOPT_CHECK(easy, CURLOPT_ACCEPT_ENCODING, "");
|
|
339
349
|
CURL_SETOPT_CHECK(easy, CURLOPT_PRIVATE, (char *)ctx);
|
|
340
350
|
CURL_SETOPT_CHECK(easy, CURLOPT_HTTP_VERSION, CURL_HTTP_VERSION_2TLS);
|
|
341
|
-
|
|
342
351
|
return CURLE_OK;
|
|
343
352
|
}
|
|
344
353
|
|
|
345
354
|
static CURLcode setup_security_options(CURL *easy) {
|
|
346
355
|
CURL_SETOPT_CHECK(easy, CURLOPT_SSL_VERIFYPEER, 1L);
|
|
347
356
|
CURL_SETOPT_CHECK(easy, CURLOPT_SSL_VERIFYHOST, 2L);
|
|
348
|
-
|
|
349
357
|
#ifdef CURLOPT_PROTOCOLS_STR
|
|
350
358
|
CURL_SETOPT_CHECK(easy, CURLOPT_PROTOCOLS_STR, "http,https");
|
|
351
359
|
CURL_SETOPT_CHECK(easy, CURLOPT_REDIR_PROTOCOLS_STR, "http,https");
|
|
@@ -353,7 +361,6 @@ static CURLcode setup_security_options(CURL *easy) {
|
|
|
353
361
|
CURL_SETOPT_CHECK(easy, CURLOPT_PROTOCOLS, CURLPROTO_HTTP | CURLPROTO_HTTPS);
|
|
354
362
|
CURL_SETOPT_CHECK(easy, CURLOPT_REDIR_PROTOCOLS, CURLPROTO_HTTP | CURLPROTO_HTTPS);
|
|
355
363
|
#endif
|
|
356
|
-
|
|
357
364
|
return CURLE_OK;
|
|
358
365
|
}
|
|
359
366
|
|
|
@@ -377,43 +384,42 @@ static CURLcode setup_method_and_body(CURL *easy, VALUE method, VALUE body) {
|
|
|
377
384
|
CURL_SETOPT_CHECK(easy, CURLOPT_POSTFIELDSIZE, RSTRING_LEN(body));
|
|
378
385
|
CURL_SETOPT_CHECK(easy, CURLOPT_COPYPOSTFIELDS, StringValuePtr(body));
|
|
379
386
|
}
|
|
380
|
-
|
|
381
387
|
return CURLE_OK;
|
|
382
388
|
}
|
|
383
389
|
|
|
384
390
|
static int header_iter_cb(VALUE key, VALUE val, VALUE arg) {
|
|
385
391
|
request_ctx_t *ctx = (request_ctx_t *)arg;
|
|
386
|
-
|
|
387
392
|
VALUE key_str = rb_String(key);
|
|
388
393
|
const char *k = RSTRING_PTR(key_str);
|
|
389
394
|
long klen = RSTRING_LEN(key_str);
|
|
390
395
|
|
|
396
|
+
if (contains_header_injection(k, klen))
|
|
397
|
+
return ST_CONTINUE;
|
|
398
|
+
|
|
391
399
|
if (NIL_P(val) || RSTRING_LEN(rb_String(val)) == 0) {
|
|
392
|
-
char
|
|
393
|
-
char *buf =
|
|
400
|
+
char sbuf[HEADER_LINE_BUF_SIZE];
|
|
401
|
+
char *buf = sbuf;
|
|
394
402
|
long need = klen + 2;
|
|
395
|
-
|
|
396
403
|
if (need > HEADER_LINE_BUF_SIZE)
|
|
397
404
|
buf = malloc(need);
|
|
398
405
|
if (!buf)
|
|
399
406
|
return ST_CONTINUE;
|
|
400
|
-
|
|
401
407
|
memcpy(buf, k, klen);
|
|
402
408
|
buf[klen] = ';';
|
|
403
409
|
buf[klen + 1] = '\0';
|
|
404
|
-
|
|
405
410
|
ctx->req_headers = curl_slist_append(ctx->req_headers, buf);
|
|
406
|
-
|
|
407
|
-
if (buf != stack_buf)
|
|
411
|
+
if (buf != sbuf)
|
|
408
412
|
free(buf);
|
|
409
413
|
} else {
|
|
410
414
|
VALUE val_str = rb_String(val);
|
|
411
415
|
const char *v = RSTRING_PTR(val_str);
|
|
412
416
|
long vlen = RSTRING_LEN(val_str);
|
|
413
|
-
char stack_buf[HEADER_LINE_BUF_SIZE];
|
|
414
|
-
char *buf = stack_buf;
|
|
415
|
-
long need = klen + 2 + vlen + 1;
|
|
416
417
|
|
|
418
|
+
if (contains_header_injection(v, vlen))
|
|
419
|
+
return ST_CONTINUE;
|
|
420
|
+
char sbuf[HEADER_LINE_BUF_SIZE];
|
|
421
|
+
char *buf = sbuf;
|
|
422
|
+
long need = klen + 2 + vlen + 1;
|
|
417
423
|
if (need > HEADER_LINE_BUF_SIZE)
|
|
418
424
|
buf = malloc(need);
|
|
419
425
|
if (!buf)
|
|
@@ -424,13 +430,10 @@ static int header_iter_cb(VALUE key, VALUE val, VALUE arg) {
|
|
|
424
430
|
buf[klen + 1] = ' ';
|
|
425
431
|
memcpy(buf + klen + 2, v, vlen);
|
|
426
432
|
buf[klen + 2 + vlen] = '\0';
|
|
427
|
-
|
|
428
433
|
ctx->req_headers = curl_slist_append(ctx->req_headers, buf);
|
|
429
|
-
|
|
430
|
-
if (buf != stack_buf)
|
|
434
|
+
if (buf != sbuf)
|
|
431
435
|
free(buf);
|
|
432
436
|
}
|
|
433
|
-
|
|
434
437
|
return ST_CONTINUE;
|
|
435
438
|
}
|
|
436
439
|
|
|
@@ -439,33 +442,25 @@ static int setup_easy_handle(request_ctx_t *ctx, VALUE request, long timeout_sec
|
|
|
439
442
|
VALUE method = rb_hash_aref(request, sym_method);
|
|
440
443
|
VALUE headers = rb_hash_aref(request, sym_headers);
|
|
441
444
|
VALUE body = rb_hash_aref(request, sym_body);
|
|
442
|
-
|
|
443
445
|
if (NIL_P(url))
|
|
444
446
|
return 0;
|
|
445
|
-
|
|
446
447
|
const char *url_str = StringValueCStr(url);
|
|
447
|
-
|
|
448
|
-
if (!is_valid_url(url_str)) {
|
|
448
|
+
if (!is_valid_url(url_str))
|
|
449
449
|
rb_raise(rb_eArgError, "Invalid URL: %s", url_str);
|
|
450
|
-
}
|
|
451
450
|
|
|
452
451
|
CURLcode res;
|
|
453
|
-
|
|
454
452
|
res = setup_basic_options(ctx->easy, url_str, timeout_sec, ctx);
|
|
455
453
|
if (res != CURLE_OK)
|
|
456
454
|
return 0;
|
|
457
|
-
|
|
458
455
|
res = setup_security_options(ctx->easy);
|
|
459
456
|
if (res != CURLE_OK)
|
|
460
457
|
return 0;
|
|
461
|
-
|
|
462
458
|
res = setup_method_and_body(ctx->easy, method, body);
|
|
463
459
|
if (res != CURLE_OK)
|
|
464
460
|
return 0;
|
|
465
461
|
|
|
466
462
|
if (!NIL_P(headers) && rb_obj_is_kind_of(headers, rb_cHash)) {
|
|
467
463
|
rb_hash_foreach(headers, header_iter_cb, (VALUE)ctx);
|
|
468
|
-
|
|
469
464
|
if (ctx->req_headers) {
|
|
470
465
|
res = curl_easy_setopt(ctx->easy, CURLOPT_HTTPHEADER, ctx->req_headers);
|
|
471
466
|
if (res != CURLE_OK)
|
|
@@ -473,46 +468,29 @@ static int setup_easy_handle(request_ctx_t *ctx, VALUE request, long timeout_sec
|
|
|
473
468
|
}
|
|
474
469
|
}
|
|
475
470
|
|
|
471
|
+
RB_GC_GUARD(url);
|
|
472
|
+
RB_GC_GUARD(method);
|
|
473
|
+
RB_GC_GUARD(headers);
|
|
474
|
+
RB_GC_GUARD(body);
|
|
475
|
+
RB_GC_GUARD(request);
|
|
476
476
|
return 1;
|
|
477
477
|
}
|
|
478
478
|
|
|
479
|
-
static void *perform_without_gvl(void *arg) {
|
|
480
|
-
multi_session_t *session = (multi_session_t *)arg;
|
|
481
|
-
|
|
482
|
-
while (session->still_running > 0) {
|
|
483
|
-
CURLMcode mc = curl_multi_perform(session->multi, &session->still_running);
|
|
484
|
-
if (mc != CURLM_OK)
|
|
485
|
-
break;
|
|
486
|
-
|
|
487
|
-
if (session->still_running > 0) {
|
|
488
|
-
int numfds = 0;
|
|
489
|
-
mc = curl_multi_poll(session->multi, NULL, 0, POLL_TIMEOUT_MS, &numfds);
|
|
490
|
-
if (mc != CURLM_OK)
|
|
491
|
-
break;
|
|
492
|
-
}
|
|
493
|
-
}
|
|
494
|
-
|
|
495
|
-
return NULL;
|
|
496
|
-
}
|
|
497
|
-
|
|
498
479
|
static void *poll_without_gvl(void *arg) {
|
|
499
|
-
multi_session_t *
|
|
480
|
+
multi_session_t *s = (multi_session_t *)arg;
|
|
481
|
+
if (s->cancelled)
|
|
482
|
+
return NULL;
|
|
500
483
|
int numfds = 0;
|
|
501
|
-
curl_multi_poll(
|
|
502
|
-
curl_multi_perform(
|
|
484
|
+
curl_multi_poll(s->multi, NULL, 0, POLL_TIMEOUT_MS, &numfds);
|
|
485
|
+
curl_multi_perform(s->multi, &s->still_running);
|
|
503
486
|
return NULL;
|
|
504
487
|
}
|
|
505
488
|
|
|
506
489
|
static void unblock_perform(void *arg) {
|
|
507
|
-
(
|
|
508
|
-
|
|
509
|
-
|
|
510
|
-
|
|
511
|
-
#ifdef HAVE_RB_FIBER_SCHEDULER_CURRENT
|
|
512
|
-
VALUE scheduler = rb_fiber_scheduler_current();
|
|
513
|
-
return scheduler != Qnil && scheduler != Qfalse;
|
|
514
|
-
#else
|
|
515
|
-
return 0;
|
|
490
|
+
multi_session_t *s = (multi_session_t *)arg;
|
|
491
|
+
s->cancelled = 1;
|
|
492
|
+
#ifdef HAVE_CURL_MULTI_WAKEUP
|
|
493
|
+
curl_multi_wakeup(s->multi);
|
|
516
494
|
#endif
|
|
517
495
|
}
|
|
518
496
|
|
|
@@ -526,97 +504,73 @@ typedef struct {
|
|
|
526
504
|
static int process_completed(multi_session_t *session, completion_ctx_t *cctx) {
|
|
527
505
|
CURLMsg *msg;
|
|
528
506
|
int msgs_left;
|
|
529
|
-
|
|
530
507
|
while ((msg = curl_multi_info_read(session->multi, &msgs_left))) {
|
|
531
508
|
if (msg->msg != CURLMSG_DONE)
|
|
532
509
|
continue;
|
|
533
|
-
|
|
534
510
|
request_ctx_t *ctx = NULL;
|
|
535
511
|
curl_easy_getinfo(msg->easy_handle, CURLINFO_PRIVATE, (char **)&ctx);
|
|
536
512
|
if (!ctx || ctx->done)
|
|
537
513
|
continue;
|
|
538
514
|
ctx->done = 1;
|
|
539
515
|
ctx->curl_result = msg->data.result;
|
|
540
|
-
if (msg->data.result == CURLE_OK)
|
|
516
|
+
if (msg->data.result == CURLE_OK)
|
|
541
517
|
curl_easy_getinfo(ctx->easy, CURLINFO_RESPONSE_CODE, &ctx->http_status);
|
|
542
|
-
}
|
|
543
|
-
|
|
544
518
|
if (cctx->stream) {
|
|
545
|
-
VALUE response
|
|
546
|
-
|
|
547
|
-
|
|
548
|
-
|
|
549
|
-
response = build_error_response_with_code(curl_easy_strerror(msg->data.result),
|
|
550
|
-
(int)msg->data.result);
|
|
551
|
-
}
|
|
519
|
+
VALUE response = (msg->data.result == CURLE_OK)
|
|
520
|
+
? build_response(ctx)
|
|
521
|
+
: build_error_response_with_code(
|
|
522
|
+
curl_easy_strerror(msg->data.result), (int)msg->data.result);
|
|
552
523
|
VALUE pair = rb_ary_new_from_args(2, INT2NUM(ctx->index), response);
|
|
553
524
|
rb_yield(pair);
|
|
554
525
|
cctx->completed++;
|
|
555
526
|
} else {
|
|
556
527
|
cctx->completed++;
|
|
557
528
|
}
|
|
558
|
-
|
|
559
529
|
if (cctx->target > 0 && cctx->completed >= cctx->target)
|
|
560
530
|
return 1;
|
|
561
531
|
}
|
|
562
|
-
|
|
563
532
|
return 0;
|
|
564
533
|
}
|
|
565
534
|
|
|
566
535
|
static void run_multi_loop(multi_session_t *session, completion_ctx_t *cctx) {
|
|
567
|
-
|
|
568
|
-
|
|
569
|
-
|
|
570
|
-
|
|
536
|
+
VALUE scheduler = current_fiber_scheduler();
|
|
537
|
+
|
|
538
|
+
if (scheduler != Qnil) {
|
|
539
|
+
curl_multi_perform(session->multi, &session->still_running);
|
|
540
|
+
while (session->still_running > 0) {
|
|
541
|
+
if (session->cancelled)
|
|
571
542
|
break;
|
|
543
|
+
run_via_fiber_worker(scheduler, poll_without_gvl, session);
|
|
572
544
|
if (process_completed(session, cctx))
|
|
573
545
|
break;
|
|
574
|
-
if (session->still_running == 0)
|
|
575
|
-
break;
|
|
576
|
-
|
|
577
|
-
int numfds = 0;
|
|
578
|
-
curl_multi_poll(session->multi, NULL, 0, FIBER_POLL_TIMEOUT_MS, &numfds);
|
|
579
|
-
rb_thread_schedule();
|
|
580
546
|
}
|
|
581
547
|
process_completed(session, cctx);
|
|
582
548
|
} else {
|
|
583
|
-
|
|
584
|
-
|
|
585
|
-
|
|
586
|
-
|
|
587
|
-
|
|
588
|
-
|
|
589
|
-
|
|
590
|
-
process_completed(session, cctx);
|
|
591
|
-
} else {
|
|
592
|
-
session->still_running = 1;
|
|
593
|
-
curl_multi_perform(session->multi, &session->still_running);
|
|
594
|
-
rb_thread_call_without_gvl(perform_without_gvl, session, unblock_perform, session);
|
|
595
|
-
process_completed(session, cctx);
|
|
549
|
+
curl_multi_perform(session->multi, &session->still_running);
|
|
550
|
+
while (session->still_running > 0) {
|
|
551
|
+
if (session->cancelled)
|
|
552
|
+
break;
|
|
553
|
+
rb_thread_call_without_gvl(poll_without_gvl, session, unblock_perform, session);
|
|
554
|
+
if (process_completed(session, cctx))
|
|
555
|
+
break;
|
|
596
556
|
}
|
|
557
|
+
process_completed(session, cctx);
|
|
597
558
|
}
|
|
598
559
|
}
|
|
599
560
|
|
|
600
561
|
static int is_default_retryable_curle(CURLcode code) {
|
|
601
|
-
for (int i = 0; i < DEFAULT_RETRYABLE_CURLE_COUNT; i++)
|
|
562
|
+
for (int i = 0; i < DEFAULT_RETRYABLE_CURLE_COUNT; i++)
|
|
602
563
|
if (DEFAULT_RETRYABLE_CURLE[i] == code)
|
|
603
564
|
return 1;
|
|
604
|
-
}
|
|
605
565
|
return 0;
|
|
606
566
|
}
|
|
607
567
|
|
|
608
|
-
static int should_retry(request_ctx_t *ctx, retry_config_t *
|
|
609
|
-
if (ctx->curl_result != CURLE_OK)
|
|
568
|
+
static int should_retry(request_ctx_t *ctx, retry_config_t *rc) {
|
|
569
|
+
if (ctx->curl_result != CURLE_OK)
|
|
610
570
|
return is_default_retryable_curle(ctx->curl_result);
|
|
611
|
-
|
|
612
|
-
|
|
613
|
-
|
|
614
|
-
for (int i = 0; i < retry_cfg->retry_http_count; i++) {
|
|
615
|
-
if (retry_cfg->retry_http_codes[i] == (int)ctx->http_status)
|
|
616
|
-
return 1;
|
|
617
|
-
}
|
|
618
|
-
}
|
|
619
|
-
|
|
571
|
+
for (int i = 0; i < rc->retry_http_count; i++)
|
|
572
|
+
if (rc->retry_http_codes[i] == (int)ctx->http_status)
|
|
573
|
+
return 1;
|
|
620
574
|
return 0;
|
|
621
575
|
}
|
|
622
576
|
|
|
@@ -626,29 +580,28 @@ typedef struct {
|
|
|
626
580
|
|
|
627
581
|
static void *sleep_without_gvl(void *arg) {
|
|
628
582
|
sleep_arg_t *sa = (sleep_arg_t *)arg;
|
|
629
|
-
struct timespec ts
|
|
630
|
-
|
|
631
|
-
ts.tv_nsec = (sa->delay_ms % 1000) * 1000000L;
|
|
583
|
+
struct timespec ts = {.tv_sec = sa->delay_ms / 1000,
|
|
584
|
+
.tv_nsec = (sa->delay_ms % 1000) * 1000000L};
|
|
632
585
|
nanosleep(&ts, NULL);
|
|
633
586
|
return NULL;
|
|
634
587
|
}
|
|
635
588
|
|
|
589
|
+
/* FIX #2: Fiber path releases GVL via run_via_fiber_worker */
|
|
636
590
|
static void retry_delay_sleep(long delay_ms) {
|
|
637
591
|
if (delay_ms <= 0)
|
|
638
592
|
return;
|
|
639
|
-
|
|
640
|
-
if (
|
|
593
|
+
VALUE scheduler = current_fiber_scheduler();
|
|
594
|
+
if (scheduler != Qnil) {
|
|
641
595
|
long remaining = delay_ms;
|
|
642
596
|
while (remaining > 0) {
|
|
643
597
|
long chunk = remaining > FIBER_POLL_TIMEOUT_MS ? FIBER_POLL_TIMEOUT_MS : remaining;
|
|
644
598
|
sleep_arg_t sa = {.delay_ms = chunk};
|
|
645
|
-
sleep_without_gvl
|
|
646
|
-
rb_thread_schedule();
|
|
599
|
+
run_via_fiber_worker(scheduler, sleep_without_gvl, &sa);
|
|
647
600
|
remaining -= chunk;
|
|
648
601
|
}
|
|
649
602
|
} else {
|
|
650
603
|
sleep_arg_t sa = {.delay_ms = delay_ms};
|
|
651
|
-
rb_thread_call_without_gvl(sleep_without_gvl, &sa,
|
|
604
|
+
rb_thread_call_without_gvl(sleep_without_gvl, &sa, (rb_unblock_function_t *)0, NULL);
|
|
652
605
|
}
|
|
653
606
|
}
|
|
654
607
|
|
|
@@ -659,50 +612,45 @@ static void parse_options(VALUE options, long *timeout, int *max_conn, retry_con
|
|
|
659
612
|
retry_cfg->retry_delay_ms = DEFAULT_RETRY_DELAY;
|
|
660
613
|
retry_cfg->retry_http_codes = NULL;
|
|
661
614
|
retry_cfg->retry_http_count = 0;
|
|
662
|
-
|
|
663
615
|
if (NIL_P(options) || !rb_obj_is_kind_of(options, rb_cHash))
|
|
664
616
|
return;
|
|
665
617
|
|
|
666
618
|
VALUE t = rb_hash_aref(options, sym_timeout);
|
|
667
619
|
if (!NIL_P(t)) {
|
|
668
|
-
long
|
|
669
|
-
if (
|
|
670
|
-
|
|
671
|
-
else if (
|
|
672
|
-
|
|
673
|
-
*timeout =
|
|
620
|
+
long v = NUM2LONG(t);
|
|
621
|
+
if (v > MAX_TIMEOUT)
|
|
622
|
+
v = MAX_TIMEOUT;
|
|
623
|
+
else if (v <= 0)
|
|
624
|
+
v = 30;
|
|
625
|
+
*timeout = v;
|
|
674
626
|
}
|
|
675
|
-
|
|
676
627
|
VALUE c = rb_hash_aref(options, sym_connections);
|
|
677
628
|
if (!NIL_P(c)) {
|
|
678
|
-
int
|
|
679
|
-
if (
|
|
680
|
-
|
|
681
|
-
else if (
|
|
682
|
-
|
|
683
|
-
*max_conn =
|
|
629
|
+
int v = NUM2INT(c);
|
|
630
|
+
if (v > 100)
|
|
631
|
+
v = 100;
|
|
632
|
+
else if (v <= 0)
|
|
633
|
+
v = 20;
|
|
634
|
+
*max_conn = v;
|
|
684
635
|
}
|
|
685
|
-
|
|
686
636
|
VALUE r = rb_hash_aref(options, sym_retries);
|
|
687
637
|
if (!NIL_P(r)) {
|
|
688
|
-
int
|
|
689
|
-
if (
|
|
690
|
-
|
|
691
|
-
if (
|
|
692
|
-
|
|
693
|
-
retry_cfg->max_retries =
|
|
638
|
+
int v = NUM2INT(r);
|
|
639
|
+
if (v < 0)
|
|
640
|
+
v = 0;
|
|
641
|
+
if (v > MAX_RETRIES)
|
|
642
|
+
v = MAX_RETRIES;
|
|
643
|
+
retry_cfg->max_retries = v;
|
|
694
644
|
}
|
|
695
|
-
|
|
696
645
|
VALUE rd = rb_hash_aref(options, sym_retry_delay);
|
|
697
646
|
if (!NIL_P(rd)) {
|
|
698
|
-
long
|
|
699
|
-
if (
|
|
700
|
-
|
|
701
|
-
if (
|
|
702
|
-
|
|
703
|
-
retry_cfg->retry_delay_ms =
|
|
647
|
+
long v = NUM2LONG(rd);
|
|
648
|
+
if (v < 0)
|
|
649
|
+
v = 0;
|
|
650
|
+
if (v > 30000)
|
|
651
|
+
v = 30000;
|
|
652
|
+
retry_cfg->retry_delay_ms = v;
|
|
704
653
|
}
|
|
705
|
-
|
|
706
654
|
VALUE rc = rb_hash_aref(options, sym_retry_codes);
|
|
707
655
|
if (!NIL_P(rc) && rb_obj_is_kind_of(rc, rb_cArray)) {
|
|
708
656
|
int len = (int)RARRAY_LEN(rc);
|
|
@@ -710,211 +658,240 @@ static void parse_options(VALUE options, long *timeout, int *max_conn, retry_con
|
|
|
710
658
|
retry_cfg->retry_http_codes = malloc(sizeof(int) * len);
|
|
711
659
|
if (retry_cfg->retry_http_codes) {
|
|
712
660
|
retry_cfg->retry_http_count = len;
|
|
713
|
-
for (int i = 0; i < len; i++)
|
|
661
|
+
for (int i = 0; i < len; i++)
|
|
714
662
|
retry_cfg->retry_http_codes[i] = NUM2INT(rb_ary_entry(rc, i));
|
|
715
|
-
}
|
|
716
663
|
}
|
|
717
664
|
}
|
|
718
665
|
}
|
|
719
666
|
}
|
|
720
667
|
|
|
721
|
-
|
|
722
|
-
|
|
723
|
-
int
|
|
724
|
-
|
|
725
|
-
|
|
726
|
-
|
|
727
|
-
|
|
728
|
-
|
|
729
|
-
|
|
730
|
-
|
|
731
|
-
|
|
732
|
-
|
|
733
|
-
|
|
734
|
-
|
|
735
|
-
|
|
736
|
-
|
|
737
|
-
if (retry_cfg.max_retries > 0 && target > 0)
|
|
738
|
-
rb_warn(
|
|
739
|
-
"FastCurl: retries are not supported in first_execute, ignoring retries option");
|
|
740
|
-
retry_cfg.max_retries = 0;
|
|
668
|
+
typedef struct {
|
|
669
|
+
multi_session_t *session;
|
|
670
|
+
int *invalid;
|
|
671
|
+
retry_config_t *retry_cfg;
|
|
672
|
+
} cleanup_ctx_t;
|
|
673
|
+
|
|
674
|
+
static VALUE cleanup_session(VALUE arg) {
|
|
675
|
+
cleanup_ctx_t *ctx = (cleanup_ctx_t *)arg;
|
|
676
|
+
if (ctx->session->requests) {
|
|
677
|
+
for (int i = 0; i < ctx->session->count; i++) {
|
|
678
|
+
if (ctx->session->requests[i].easy)
|
|
679
|
+
curl_multi_remove_handle(ctx->session->multi, ctx->session->requests[i].easy);
|
|
680
|
+
request_ctx_free(&ctx->session->requests[i]);
|
|
681
|
+
}
|
|
682
|
+
free(ctx->session->requests);
|
|
683
|
+
ctx->session->requests = NULL;
|
|
741
684
|
}
|
|
742
|
-
|
|
743
|
-
|
|
744
|
-
|
|
745
|
-
session.count = count;
|
|
746
|
-
session.timeout_ms = timeout_sec * 1000;
|
|
747
|
-
session.max_connections = max_conn;
|
|
748
|
-
|
|
749
|
-
curl_multi_setopt(session.multi, CURLMOPT_MAXCONNECTS, (long)max_conn);
|
|
750
|
-
curl_multi_setopt(session.multi, CURLMOPT_MAX_TOTAL_CONNECTIONS, (long)max_conn);
|
|
751
|
-
#ifdef CURLPIPE_MULTIPLEX
|
|
752
|
-
curl_multi_setopt(session.multi, CURLMOPT_PIPELINING, CURLPIPE_MULTIPLEX);
|
|
753
|
-
#endif
|
|
754
|
-
|
|
755
|
-
session.requests = calloc(count, sizeof(request_ctx_t));
|
|
756
|
-
if (!session.requests) {
|
|
757
|
-
curl_multi_cleanup(session.multi);
|
|
758
|
-
if (retry_cfg.retry_http_codes)
|
|
759
|
-
free(retry_cfg.retry_http_codes);
|
|
760
|
-
rb_raise(rb_eNoMemError, "failed to allocate request contexts");
|
|
685
|
+
if (ctx->invalid) {
|
|
686
|
+
free(ctx->invalid);
|
|
687
|
+
ctx->invalid = NULL;
|
|
761
688
|
}
|
|
762
|
-
|
|
763
|
-
|
|
764
|
-
|
|
765
|
-
|
|
766
|
-
|
|
767
|
-
|
|
768
|
-
|
|
769
|
-
rb_raise(rb_eNoMemError, "failed to allocate tracking array");
|
|
689
|
+
if (ctx->session->multi) {
|
|
690
|
+
curl_multi_cleanup(ctx->session->multi);
|
|
691
|
+
ctx->session->multi = NULL;
|
|
692
|
+
}
|
|
693
|
+
if (ctx->retry_cfg && ctx->retry_cfg->retry_http_codes) {
|
|
694
|
+
free(ctx->retry_cfg->retry_http_codes);
|
|
695
|
+
ctx->retry_cfg->retry_http_codes = NULL;
|
|
770
696
|
}
|
|
697
|
+
return Qnil;
|
|
698
|
+
}
|
|
699
|
+
|
|
700
|
+
typedef struct {
|
|
701
|
+
VALUE requests;
|
|
702
|
+
VALUE options;
|
|
703
|
+
int target;
|
|
704
|
+
int stream;
|
|
705
|
+
multi_session_t *session;
|
|
706
|
+
int *invalid;
|
|
707
|
+
retry_config_t *retry_cfg;
|
|
708
|
+
long timeout_sec;
|
|
709
|
+
} execute_args_t;
|
|
710
|
+
|
|
711
|
+
static VALUE internal_execute_body(VALUE arg) {
|
|
712
|
+
execute_args_t *ea = (execute_args_t *)arg;
|
|
713
|
+
VALUE requests = ea->requests;
|
|
714
|
+
multi_session_t *session = ea->session;
|
|
715
|
+
int *invalid = ea->invalid;
|
|
716
|
+
retry_config_t *retry_cfg = ea->retry_cfg;
|
|
717
|
+
long timeout_sec = ea->timeout_sec;
|
|
718
|
+
int count = session->count, target = ea->target, stream = ea->stream;
|
|
771
719
|
|
|
772
720
|
int valid_requests = 0;
|
|
773
721
|
for (int i = 0; i < count; i++) {
|
|
774
722
|
VALUE req = rb_ary_entry(requests, i);
|
|
775
|
-
request_ctx_init(&session
|
|
776
|
-
|
|
777
|
-
|
|
778
|
-
session.requests[i].done = 1;
|
|
723
|
+
request_ctx_init(&session->requests[i], i);
|
|
724
|
+
if (!setup_easy_handle(&session->requests[i], req, timeout_sec)) {
|
|
725
|
+
session->requests[i].done = 1;
|
|
779
726
|
invalid[i] = 1;
|
|
780
727
|
continue;
|
|
781
728
|
}
|
|
782
|
-
|
|
783
|
-
CURLMcode mc = curl_multi_add_handle(session.multi, session.requests[i].easy);
|
|
729
|
+
CURLMcode mc = curl_multi_add_handle(session->multi, session->requests[i].easy);
|
|
784
730
|
if (mc != CURLM_OK) {
|
|
785
|
-
session
|
|
731
|
+
session->requests[i].done = 1;
|
|
786
732
|
invalid[i] = 1;
|
|
787
733
|
continue;
|
|
788
734
|
}
|
|
789
|
-
|
|
790
735
|
valid_requests++;
|
|
791
736
|
}
|
|
792
|
-
|
|
793
737
|
if (valid_requests == 0)
|
|
794
|
-
session
|
|
738
|
+
session->still_running = 0;
|
|
795
739
|
|
|
796
740
|
completion_ctx_t cctx;
|
|
797
741
|
cctx.results = stream ? Qnil : rb_ary_new2(count);
|
|
798
742
|
cctx.completed = 0;
|
|
799
743
|
cctx.target = target;
|
|
800
744
|
cctx.stream = stream;
|
|
801
|
-
|
|
802
745
|
if (!stream) {
|
|
803
746
|
for (int i = 0; i < count; i++)
|
|
804
747
|
rb_ary_store(cctx.results, i, Qnil);
|
|
805
748
|
}
|
|
806
749
|
|
|
807
|
-
run_multi_loop(
|
|
750
|
+
run_multi_loop(session, &cctx);
|
|
808
751
|
|
|
809
|
-
|
|
810
|
-
if (!stream && retry_cfg.max_retries > 0) {
|
|
752
|
+
if (!stream && retry_cfg->max_retries > 0) {
|
|
811
753
|
int prev_all_failed = 0;
|
|
812
|
-
|
|
813
|
-
for (int attempt = 0; attempt < retry_cfg.max_retries; attempt++) {
|
|
754
|
+
for (int attempt = 0; attempt < retry_cfg->max_retries; attempt++) {
|
|
814
755
|
int retry_count = 0;
|
|
815
|
-
int *
|
|
816
|
-
if (!
|
|
756
|
+
int *ri = malloc(sizeof(int) * count);
|
|
757
|
+
if (!ri)
|
|
817
758
|
break;
|
|
818
|
-
|
|
819
759
|
for (int i = 0; i < count; i++) {
|
|
820
|
-
if (invalid[i])
|
|
760
|
+
if (invalid[i] || !session->requests[i].done)
|
|
821
761
|
continue;
|
|
822
|
-
if (
|
|
823
|
-
|
|
824
|
-
if (should_retry(&session.requests[i], &retry_cfg)) {
|
|
825
|
-
retry_indices[retry_count++] = i;
|
|
826
|
-
}
|
|
762
|
+
if (should_retry(&session->requests[i], retry_cfg))
|
|
763
|
+
ri[retry_count++] = i;
|
|
827
764
|
}
|
|
828
|
-
|
|
829
765
|
if (retry_count == 0) {
|
|
830
|
-
free(
|
|
766
|
+
free(ri);
|
|
831
767
|
break;
|
|
832
768
|
}
|
|
833
|
-
|
|
834
769
|
int done_count = 0;
|
|
835
|
-
for (int i = 0; i < count; i++)
|
|
836
|
-
if (!invalid[i] && session
|
|
770
|
+
for (int i = 0; i < count; i++)
|
|
771
|
+
if (!invalid[i] && session->requests[i].done)
|
|
837
772
|
done_count++;
|
|
838
|
-
|
|
839
|
-
|
|
840
|
-
|
|
841
|
-
|
|
842
|
-
if (all_failed_this_round && prev_all_failed) {
|
|
843
|
-
free(retry_indices);
|
|
773
|
+
int all_failed = (retry_count == done_count);
|
|
774
|
+
if (all_failed && prev_all_failed) {
|
|
775
|
+
free(ri);
|
|
844
776
|
break;
|
|
845
777
|
}
|
|
846
|
-
|
|
847
|
-
|
|
848
|
-
|
|
849
|
-
retry_delay_sleep(retry_cfg.retry_delay_ms);
|
|
850
|
-
|
|
778
|
+
prev_all_failed = all_failed;
|
|
779
|
+
retry_delay_sleep(retry_cfg->retry_delay_ms);
|
|
851
780
|
for (int r = 0; r < retry_count; r++) {
|
|
852
|
-
int idx =
|
|
853
|
-
request_ctx_t *
|
|
854
|
-
|
|
855
|
-
|
|
856
|
-
|
|
857
|
-
if (!request_ctx_reset_for_retry(ctx)) {
|
|
858
|
-
ctx->done = 1;
|
|
781
|
+
int idx = ri[r];
|
|
782
|
+
request_ctx_t *rc = &session->requests[idx];
|
|
783
|
+
curl_multi_remove_handle(session->multi, rc->easy);
|
|
784
|
+
if (!request_ctx_reset_for_retry(rc)) {
|
|
785
|
+
rc->done = 1;
|
|
859
786
|
invalid[idx] = 1;
|
|
860
787
|
continue;
|
|
861
788
|
}
|
|
862
|
-
|
|
863
789
|
VALUE req = rb_ary_entry(requests, idx);
|
|
864
|
-
if (!setup_easy_handle(
|
|
865
|
-
|
|
790
|
+
if (!setup_easy_handle(rc, req, timeout_sec)) {
|
|
791
|
+
rc->done = 1;
|
|
866
792
|
invalid[idx] = 1;
|
|
867
793
|
continue;
|
|
868
794
|
}
|
|
869
|
-
|
|
870
|
-
CURLMcode mc = curl_multi_add_handle(session.multi, ctx->easy);
|
|
795
|
+
CURLMcode mc = curl_multi_add_handle(session->multi, rc->easy);
|
|
871
796
|
if (mc != CURLM_OK) {
|
|
872
|
-
|
|
797
|
+
rc->done = 1;
|
|
873
798
|
invalid[idx] = 1;
|
|
874
799
|
}
|
|
875
800
|
}
|
|
876
|
-
|
|
877
|
-
free(retry_indices);
|
|
878
|
-
|
|
801
|
+
free(ri);
|
|
879
802
|
cctx.completed = 0;
|
|
880
|
-
run_multi_loop(
|
|
803
|
+
run_multi_loop(session, &cctx);
|
|
881
804
|
}
|
|
882
805
|
}
|
|
883
806
|
|
|
884
807
|
if (!stream) {
|
|
885
808
|
for (int i = 0; i < count; i++) {
|
|
886
|
-
request_ctx_t *
|
|
887
|
-
|
|
888
|
-
if (invalid[i]) {
|
|
889
|
-
VALUE error_response = build_error_response("Invalid request configuration");
|
|
890
|
-
VALUE pair = rb_ary_new_from_args(2, INT2NUM(i), error_response);
|
|
891
|
-
rb_ary_store(cctx.results, i, pair);
|
|
892
|
-
continue;
|
|
893
|
-
}
|
|
894
|
-
|
|
809
|
+
request_ctx_t *rc = &session->requests[i];
|
|
895
810
|
VALUE response;
|
|
896
|
-
if (
|
|
897
|
-
response =
|
|
811
|
+
if (invalid[i]) {
|
|
812
|
+
response = build_error_response("Invalid request configuration");
|
|
813
|
+
} else if (rc->curl_result == CURLE_OK) {
|
|
814
|
+
response = build_response(rc);
|
|
898
815
|
} else {
|
|
899
|
-
response = build_error_response_with_code(curl_easy_strerror(
|
|
900
|
-
(int)
|
|
816
|
+
response = build_error_response_with_code(curl_easy_strerror(rc->curl_result),
|
|
817
|
+
(int)rc->curl_result);
|
|
901
818
|
}
|
|
902
|
-
|
|
903
|
-
rb_ary_store(cctx.results, i, pair);
|
|
819
|
+
rb_ary_store(cctx.results, i, rb_ary_new_from_args(2, INT2NUM(i), response));
|
|
904
820
|
}
|
|
905
821
|
}
|
|
822
|
+
return stream ? Qnil : cctx.results;
|
|
823
|
+
}
|
|
906
824
|
|
|
907
|
-
|
|
908
|
-
|
|
909
|
-
|
|
825
|
+
static VALUE internal_execute(VALUE requests, VALUE options, int target, int stream) {
|
|
826
|
+
Check_Type(requests, T_ARRAY);
|
|
827
|
+
|
|
828
|
+
long count_long = RARRAY_LEN(requests);
|
|
829
|
+
if (count_long == 0)
|
|
830
|
+
return rb_ary_new();
|
|
831
|
+
if (count_long > MAX_REQUESTS)
|
|
832
|
+
rb_raise(rb_eArgError, "too many requests (%ld), maximum is %d", count_long, MAX_REQUESTS);
|
|
833
|
+
if (count_long > INT_MAX)
|
|
834
|
+
rb_raise(rb_eArgError, "request count overflows int");
|
|
835
|
+
int count = (int)count_long;
|
|
836
|
+
|
|
837
|
+
long timeout_sec;
|
|
838
|
+
int max_conn;
|
|
839
|
+
retry_config_t retry_cfg;
|
|
840
|
+
parse_options(options, &timeout_sec, &max_conn, &retry_cfg);
|
|
841
|
+
|
|
842
|
+
if (stream || target > 0) {
|
|
843
|
+
if (retry_cfg.max_retries > 0 && stream)
|
|
844
|
+
rb_warn("FastCurl: retries are not supported in stream_execute, ignoring "
|
|
845
|
+
"retries option");
|
|
846
|
+
if (retry_cfg.max_retries > 0 && target > 0)
|
|
847
|
+
rb_warn("FastCurl: retries are not supported in first_execute, ignoring "
|
|
848
|
+
"retries option");
|
|
849
|
+
retry_cfg.max_retries = 0;
|
|
910
850
|
}
|
|
911
|
-
free(session.requests);
|
|
912
|
-
free(invalid);
|
|
913
|
-
curl_multi_cleanup(session.multi);
|
|
914
|
-
if (retry_cfg.retry_http_codes)
|
|
915
|
-
free(retry_cfg.retry_http_codes);
|
|
916
851
|
|
|
917
|
-
|
|
852
|
+
multi_session_t session;
|
|
853
|
+
session.multi = curl_multi_init();
|
|
854
|
+
session.count = count;
|
|
855
|
+
session.timeout_ms = timeout_sec * 1000;
|
|
856
|
+
session.max_connections = max_conn;
|
|
857
|
+
session.cancelled = 0;
|
|
858
|
+
session.requests = NULL;
|
|
859
|
+
|
|
860
|
+
curl_multi_setopt(session.multi, CURLMOPT_MAXCONNECTS, (long)max_conn);
|
|
861
|
+
curl_multi_setopt(session.multi, CURLMOPT_MAX_TOTAL_CONNECTIONS, (long)max_conn);
|
|
862
|
+
#ifdef CURLPIPE_MULTIPLEX
|
|
863
|
+
curl_multi_setopt(session.multi, CURLMOPT_PIPELINING, CURLPIPE_MULTIPLEX);
|
|
864
|
+
#endif
|
|
865
|
+
|
|
866
|
+
session.requests = calloc(count, sizeof(request_ctx_t));
|
|
867
|
+
if (!session.requests) {
|
|
868
|
+
curl_multi_cleanup(session.multi);
|
|
869
|
+
if (retry_cfg.retry_http_codes)
|
|
870
|
+
free(retry_cfg.retry_http_codes);
|
|
871
|
+
rb_raise(rb_eNoMemError, "failed to allocate request contexts");
|
|
872
|
+
}
|
|
873
|
+
|
|
874
|
+
int *invalid = calloc(count, sizeof(int));
|
|
875
|
+
if (!invalid) {
|
|
876
|
+
free(session.requests);
|
|
877
|
+
curl_multi_cleanup(session.multi);
|
|
878
|
+
if (retry_cfg.retry_http_codes)
|
|
879
|
+
free(retry_cfg.retry_http_codes);
|
|
880
|
+
rb_raise(rb_eNoMemError, "failed to allocate tracking array");
|
|
881
|
+
}
|
|
882
|
+
|
|
883
|
+
cleanup_ctx_t cleanup = {.session = &session, .invalid = invalid, .retry_cfg = &retry_cfg};
|
|
884
|
+
execute_args_t ea = {
|
|
885
|
+
.requests = requests,
|
|
886
|
+
.options = options,
|
|
887
|
+
.target = target,
|
|
888
|
+
.stream = stream,
|
|
889
|
+
.session = &session,
|
|
890
|
+
.invalid = invalid,
|
|
891
|
+
.retry_cfg = &retry_cfg,
|
|
892
|
+
.timeout_sec = timeout_sec,
|
|
893
|
+
};
|
|
894
|
+
return rb_ensure(internal_execute_body, (VALUE)&ea, cleanup_session, (VALUE)&cleanup);
|
|
918
895
|
}
|
|
919
896
|
|
|
920
897
|
static VALUE rb_fast_curl_execute(int argc, VALUE *argv, VALUE self) {
|
|
@@ -926,24 +903,20 @@ static VALUE rb_fast_curl_execute(int argc, VALUE *argv, VALUE self) {
|
|
|
926
903
|
static VALUE rb_fast_curl_first_execute(int argc, VALUE *argv, VALUE self) {
|
|
927
904
|
VALUE requests, options;
|
|
928
905
|
rb_scan_args(argc, argv, "1:", &requests, &options);
|
|
929
|
-
|
|
930
906
|
int count = 1;
|
|
931
907
|
if (!NIL_P(options)) {
|
|
932
908
|
VALUE c = rb_hash_aref(options, sym_count);
|
|
933
909
|
if (!NIL_P(c))
|
|
934
910
|
count = NUM2INT(c);
|
|
935
911
|
}
|
|
936
|
-
|
|
937
912
|
return internal_execute(requests, options, count, 0);
|
|
938
913
|
}
|
|
939
914
|
|
|
940
915
|
static VALUE rb_fast_curl_stream_execute(int argc, VALUE *argv, VALUE self) {
|
|
941
916
|
VALUE requests, options;
|
|
942
917
|
rb_scan_args(argc, argv, "1:", &requests, &options);
|
|
943
|
-
|
|
944
918
|
if (!rb_block_given_p())
|
|
945
919
|
rb_raise(rb_eArgError, "stream_execute requires a block");
|
|
946
|
-
|
|
947
920
|
return internal_execute(requests, options, -1, 1);
|
|
948
921
|
}
|
|
949
922
|
|
data/lib/fast_curl/version.rb
CHANGED
metadata
CHANGED
|
@@ -1,14 +1,14 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: fast_curl
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 0.
|
|
4
|
+
version: 0.3.0
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- roman-haidarov
|
|
8
8
|
autorequire:
|
|
9
9
|
bindir: bin
|
|
10
10
|
cert_chain: []
|
|
11
|
-
date: 2026-
|
|
11
|
+
date: 2026-04-15 00:00:00.000000000 Z
|
|
12
12
|
dependencies:
|
|
13
13
|
- !ruby/object:Gem::Dependency
|
|
14
14
|
name: json
|
|
@@ -123,7 +123,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
|
|
|
123
123
|
requirements:
|
|
124
124
|
- - ">="
|
|
125
125
|
- !ruby/object:Gem::Version
|
|
126
|
-
version: 3.
|
|
126
|
+
version: 3.1.0
|
|
127
127
|
required_rubygems_version: !ruby/object:Gem::Requirement
|
|
128
128
|
requirements:
|
|
129
129
|
- - ">="
|