http_spew 0.1.0 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/.gitignore +3 -0
- data/.manifest +6 -0
- data/ChangeLog +303 -0
- data/GIT-VERSION-FILE +1 -1
- data/GIT-VERSION-GEN +1 -1
- data/GNUmakefile +11 -0
- data/Gemfile +5 -0
- data/LATEST +5 -2
- data/NEWS +7 -0
- data/http_spew.gemspec +2 -2
- data/lib/http_spew/chunky_pipe.rb +18 -4
- data/lib/http_spew/class_methods.rb +125 -0
- data/lib/http_spew/content_md5.rb +36 -26
- data/lib/http_spew/headers.rb +11 -0
- data/lib/http_spew/hit_n_run.rb +2 -0
- data/lib/http_spew/input_spray.rb +19 -27
- data/lib/http_spew/request.rb +65 -11
- data/lib/http_spew/version.rb +2 -0
- data/lib/http_spew.rb +13 -83
- data/pkg.mk +4 -0
- data/test/content-md5.ru +3 -1
- data/test/helper.rb +4 -1
- data/test/response_code.ru +20 -0
- data/test/test_content_md5.rb +14 -13
- data/test/test_input_spray.rb +12 -13
- data/test/test_input_spray_with_md5.rb +107 -0
- data/test/test_mirror.rb +7 -7
- data/test/test_request.rb +2 -2
- data/test/test_unexpected_response.rb +47 -0
- data/test/test_upload.rb +21 -19
- metadata +20 -13
data/.gitignore
CHANGED
data/.manifest
CHANGED
@@ -7,6 +7,7 @@ ChangeLog
|
|
7
7
|
GIT-VERSION-FILE
|
8
8
|
GIT-VERSION-GEN
|
9
9
|
GNUmakefile
|
10
|
+
Gemfile
|
10
11
|
LATEST
|
11
12
|
LICENSE
|
12
13
|
NEWS
|
@@ -14,20 +15,25 @@ README
|
|
14
15
|
http_spew.gemspec
|
15
16
|
lib/http_spew.rb
|
16
17
|
lib/http_spew/chunky_pipe.rb
|
18
|
+
lib/http_spew/class_methods.rb
|
17
19
|
lib/http_spew/content_md5.rb
|
18
20
|
lib/http_spew/headers.rb
|
19
21
|
lib/http_spew/hit_n_run.rb
|
20
22
|
lib/http_spew/input_spray.rb
|
21
23
|
lib/http_spew/request.rb
|
24
|
+
lib/http_spew/version.rb
|
22
25
|
pkg.mk
|
23
26
|
setup.rb
|
24
27
|
test/content-md5.ru
|
25
28
|
test/helper.rb
|
26
29
|
test/mirror.ru
|
30
|
+
test/response_code.ru
|
27
31
|
test/sha1.ru
|
28
32
|
test/test_content_md5.rb
|
29
33
|
test/test_hit_n_run.rb
|
30
34
|
test/test_input_spray.rb
|
35
|
+
test/test_input_spray_with_md5.rb
|
31
36
|
test/test_mirror.rb
|
32
37
|
test/test_request.rb
|
38
|
+
test/test_unexpected_response.rb
|
33
39
|
test/test_upload.rb
|
data/ChangeLog
CHANGED
@@ -1,5 +1,308 @@
|
|
1
1
|
ChangeLog from http://bogomips.org/http_spew.git
|
2
2
|
|
3
|
+
commit 84eaf4df3da4ef55e21649b971f2f246ab556b52
|
4
|
+
Author: Eric Wong <normalperson@yhbt.net>
|
5
|
+
Date: Tue May 10 14:40:53 2011 -0700
|
6
|
+
|
7
|
+
http_spew 0.2.0 - bugfixes and improvements
|
8
|
+
|
9
|
+
InputSpray may now be layered on top of ContentMD5 successfully.
|
10
|
+
Improved multi-threading abilities and tests.
|
11
|
+
|
12
|
+
Do not consider the API remotely stable, yet.
|
13
|
+
|
14
|
+
commit 02e333f025fb31e3384941c8c890b94394f98c34
|
15
|
+
Author: Eric Wong <normalperson@yhbt.net>
|
16
|
+
Date: Mon May 9 12:07:14 2011 -0700
|
17
|
+
|
18
|
+
test: disable rewindable input for Unicorn
|
19
|
+
|
20
|
+
In most cases, it's a feature we do not want.
|
21
|
+
|
22
|
+
commit c4b3c9d4231655b053472d47046d5512e55a79a9
|
23
|
+
Author: Eric Wong <normalperson@yhbt.net>
|
24
|
+
Date: Mon May 9 19:03:27 2011 +0000
|
25
|
+
|
26
|
+
request: fix bad constant reference
|
27
|
+
|
28
|
+
Oops :<
|
29
|
+
|
30
|
+
commit 8849da940687bd6433cf4b18cb4096f6867f3b3e
|
31
|
+
Author: Eric Wong <normalperson@yhbt.net>
|
32
|
+
Date: Mon May 9 18:32:28 2011 +0000
|
33
|
+
|
34
|
+
content_md5: switch back to pipe for tee-ability
|
35
|
+
|
36
|
+
This will save memory bandwidth down the line on Linux
|
37
|
+
|
38
|
+
commit 8c47ed775e7d40291660354bff29efea3d6f177a
|
39
|
+
Author: Eric Wong <normalperson@yhbt.net>
|
40
|
+
Date: Sat May 7 02:20:28 2011 +0000
|
41
|
+
|
42
|
+
content_md5: small garbage reduction
|
43
|
+
|
44
|
+
Might help somewhere...
|
45
|
+
|
46
|
+
commit ec0ff47df8c4068da2977cc9cb444caaac17b571
|
47
|
+
Author: Eric Wong <normalperson@yhbt.net>
|
48
|
+
Date: Sat May 7 01:52:16 2011 +0000
|
49
|
+
|
50
|
+
improve reliability of input_spray
|
51
|
+
|
52
|
+
Use pipes to avoid race condidions and deadlocks
|
53
|
+
|
54
|
+
commit 8157576344b9fcf98e85aafcc958c6ebe385b55b
|
55
|
+
Author: Eric Wong <normalperson@yhbt.net>
|
56
|
+
Date: Fri May 6 20:38:43 2011 +0000
|
57
|
+
|
58
|
+
update to kgio 2.4.0
|
59
|
+
|
60
|
+
Kgio.poll no longer returns EINTR
|
61
|
+
|
62
|
+
commit af2462a69eae821d6160fd2c2e571a589088977c
|
63
|
+
Author: Eric Wong <normalperson@yhbt.net>
|
64
|
+
Date: Thu May 5 16:53:53 2011 -0700
|
65
|
+
|
66
|
+
chunky_pipe: better error checking
|
67
|
+
|
68
|
+
commit 58ec2331a74ae472694fddae03f4ff454a801444
|
69
|
+
Author: Eric Wong <normalperson@yhbt.net>
|
70
|
+
Date: Thu May 5 16:53:04 2011 -0700
|
71
|
+
|
72
|
+
test_request: longer timeout for tests
|
73
|
+
|
74
|
+
This increases reliability, hopefully
|
75
|
+
|
76
|
+
commit 014cfa31df238a6b7a7e1652442613eed841b757
|
77
|
+
Author: Eric Wong <normalperson@yhbt.net>
|
78
|
+
Date: Wed May 4 18:11:22 2011 -0700
|
79
|
+
|
80
|
+
class_methods: fix race/deadlock issues
|
81
|
+
|
82
|
+
commit 254b6917d15d4bc013d9ca6889dc802b41592903
|
83
|
+
Author: Eric Wong <normalperson@yhbt.net>
|
84
|
+
Date: Wed Apr 20 17:53:14 2011 -0700
|
85
|
+
|
86
|
+
GNUmakefile: version file is added to gem
|
87
|
+
|
88
|
+
It's kinda useless otherwise...
|
89
|
+
|
90
|
+
commit 20fc09850031fb0a26f71ceafd15c93b924582f9
|
91
|
+
Author: Eric Wong <normalperson@yhbt.net>
|
92
|
+
Date: Wed Apr 20 17:44:01 2011 -0700
|
93
|
+
|
94
|
+
content_md5: stop using a pipe + thread
|
95
|
+
|
96
|
+
Too complex.
|
97
|
+
|
98
|
+
commit 7ac87f270c2583262ea923960c7cbb8a9b1f2dea
|
99
|
+
Author: Eric Wong <normalperson@yhbt.net>
|
100
|
+
Date: Wed Apr 20 17:43:41 2011 -0700
|
101
|
+
|
102
|
+
test_upload: more robust test case
|
103
|
+
|
104
|
+
Servers can be super fast.
|
105
|
+
|
106
|
+
commit 5e7e0553b7b435940afd478b83ebc265f31fdcf3
|
107
|
+
Author: Eric Wong <normalperson@yhbt.net>
|
108
|
+
Date: Wed Apr 20 17:43:13 2011 -0700
|
109
|
+
|
110
|
+
test_upload: fix timeout for seconds
|
111
|
+
|
112
|
+
HTTP_Spew.wait is now in seconds, not milliseconds
|
113
|
+
|
114
|
+
commit 33ae61959c657b5c60c2b7d9adb137ad46b24a15
|
115
|
+
Author: Eric Wong <normalperson@yhbt.net>
|
116
|
+
Date: Wed Apr 20 16:53:13 2011 -0700
|
117
|
+
|
118
|
+
request: handle EOF on read properly
|
119
|
+
|
120
|
+
"eof!" was never a method :x
|
121
|
+
|
122
|
+
commit f524b02e638cf6ec138f92538cf169edf524326c
|
123
|
+
Author: Eric Wong <normalperson@yhbt.net>
|
124
|
+
Date: Wed Apr 20 16:52:34 2011 -0700
|
125
|
+
|
126
|
+
content_md5: use kgio_write instead of IO#write
|
127
|
+
|
128
|
+
for consistency, not that we really take advantage
|
129
|
+
of kgio_write at the moment...
|
130
|
+
|
131
|
+
commit 3dd4f8c357b687bc1bf47785253e84cd7caa4661
|
132
|
+
Author: Eric Wong <normalperson@yhbt.net>
|
133
|
+
Date: Wed Apr 20 16:52:07 2011 -0700
|
134
|
+
|
135
|
+
input_spray: kill needless ivar assignment
|
136
|
+
|
137
|
+
Nothing references the writer thread.
|
138
|
+
|
139
|
+
commit b3dc72f3762b737bfc5dbed2d3d296a61879dc09
|
140
|
+
Author: Eric Wong <normalperson@yhbt.net>
|
141
|
+
Date: Wed Apr 20 16:51:23 2011 -0700
|
142
|
+
|
143
|
+
test/helper: use Thread.abort_on_exception = true
|
144
|
+
|
145
|
+
We want to detect failures during testing and fail hard
|
146
|
+
|
147
|
+
commit bde96ebe0c31508a4128542fce8753ba5e7403d7
|
148
|
+
Author: Eric Wong <normalperson@yhbt.net>
|
149
|
+
Date: Wed Apr 20 16:09:18 2011 -0700
|
150
|
+
|
151
|
+
remove timed_queue class
|
152
|
+
|
153
|
+
We don't need it, a mutex is all we need.
|
154
|
+
|
155
|
+
commit 9e5c03f84e4961d236398d416d526eef8344f98f
|
156
|
+
Author: Eric Wong <normalperson@yhbt.net>
|
157
|
+
Date: Wed Apr 20 15:42:53 2011 -0700
|
158
|
+
|
159
|
+
auto-generate version.rb file based on git version
|
160
|
+
|
161
|
+
It's helpful for people using in-development versions
|
162
|
+
of the code.
|
163
|
+
|
164
|
+
commit 6657070f27364688bba503c7e58add6dfe4dd8bf
|
165
|
+
Author: Eric Wong <normalperson@yhbt.net>
|
166
|
+
Date: Wed Apr 20 15:34:20 2011 -0700
|
167
|
+
|
168
|
+
reorganize class methods and exceptions
|
169
|
+
|
170
|
+
Also none of the exceptions we raise are for programming/user
|
171
|
+
so don't bother generating an expensive backtrace.
|
172
|
+
|
173
|
+
commit 100c23e10910bb8daff5507fd69f5d951b804672
|
174
|
+
Author: Eric Wong <normalperson@yhbt.net>
|
175
|
+
Date: Wed Apr 20 15:12:27 2011 -0700
|
176
|
+
|
177
|
+
request: support limiting accepted response codes
|
178
|
+
|
179
|
+
Some apps don't want certain responses and we won't
|
180
|
+
treat that as valid.
|
181
|
+
|
182
|
+
commit 67e23586d908eeed510508405dcdbee1dc5f9033
|
183
|
+
Author: Eric Wong <normalperson@yhbt.net>
|
184
|
+
Date: Wed Apr 20 14:49:06 2011 -0700
|
185
|
+
|
186
|
+
wait_mt: simplify
|
187
|
+
|
188
|
+
commit d2e882bdf763319ab149044440fe9bf108d46bc4
|
189
|
+
Author: Eric Wong <normalperson@yhbt.net>
|
190
|
+
Date: Wed Apr 20 14:29:26 2011 -0700
|
191
|
+
|
192
|
+
timed_queue: fix unused variable warning
|
193
|
+
|
194
|
+
commit 87cb2f5328bc20245a53a8462e6c1324c977189e
|
195
|
+
Author: Eric Wong <normalperson@yhbt.net>
|
196
|
+
Date: Wed Apr 20 14:17:20 2011 -0700
|
197
|
+
|
198
|
+
content_md5: make this a read-only IO-like object
|
199
|
+
|
200
|
+
It's easier to use this way since we'll also be able
|
201
|
+
to see the Content-MD5 and number of bytes digested.
|
202
|
+
|
203
|
+
commit 4a3f8cae93d12ebea896227e789d01912a3d55f3
|
204
|
+
Author: Eric Wong <normalperson@yhbt.net>
|
205
|
+
Date: Wed Apr 20 14:01:39 2011 -0700
|
206
|
+
|
207
|
+
chunky_pipe: autoclose on EOF
|
208
|
+
|
209
|
+
Since env["rack.input"] can't reliably be closed any
|
210
|
+
other way.
|
211
|
+
|
212
|
+
commit 0b6c35740ab1dd0ac64183b9ae0f1a5c710e4022
|
213
|
+
Author: Eric Wong <normalperson@yhbt.net>
|
214
|
+
Date: Thu Apr 14 15:19:02 2011 -0700
|
215
|
+
|
216
|
+
timeouts are in seconds, not milliseconds
|
217
|
+
|
218
|
+
It's more consistent with other Ruby code. poll(2)
|
219
|
+
(and epoll(2)) remain exception to that, so the
|
220
|
+
Ruby wrappers for those calls still use milliseconds.
|
221
|
+
|
222
|
+
commit 9e727f32a6e3506994934b9a48204ffdf0003bca
|
223
|
+
Author: Eric Wong <normalperson@yhbt.net>
|
224
|
+
Date: Thu Apr 14 21:56:48 2011 +0000
|
225
|
+
|
226
|
+
allow combining input_spray + MD5 filter
|
227
|
+
|
228
|
+
This is required for large parallel uploads.
|
229
|
+
|
230
|
+
commit 3aa054616bae3184ead01e78d70a707b59f71e63
|
231
|
+
Author: Eric Wong <normalperson@yhbt.net>
|
232
|
+
Date: Thu Apr 14 20:59:59 2011 +0000
|
233
|
+
|
234
|
+
Gemfile: depend on Unicorn 3 for tests
|
235
|
+
|
236
|
+
We need trailer support
|
237
|
+
|
238
|
+
commit b35f0d9bcdd3551bcdf2b5285c92197767fde496
|
239
|
+
Author: Eric Wong <normalperson@yhbt.net>
|
240
|
+
Date: Fri Apr 8 06:15:04 2011 +0000
|
241
|
+
|
242
|
+
timed_queue: Queue-like class that works with a timeout
|
243
|
+
|
244
|
+
commit 3db374e644014726092913eee7a4f09d15f6762c
|
245
|
+
Author: Eric Wong <normalperson@yhbt.net>
|
246
|
+
Date: Tue Apr 5 17:37:54 2011 -0700
|
247
|
+
|
248
|
+
chunky_pipe: rdoc purpose of this class
|
249
|
+
|
250
|
+
commit aac364b03306b1ba6ac65c325d946db4b0af02bd
|
251
|
+
Author: Eric Wong <normalperson@yhbt.net>
|
252
|
+
Date: Tue Apr 5 17:34:01 2011 -0700
|
253
|
+
|
254
|
+
content_md5: cleaner pack invocation (Ruby 1.9-only)
|
255
|
+
|
256
|
+
No need to String#strip!
|
257
|
+
|
258
|
+
commit c78c63adde85f0b329b44484ae652b0acdee4063
|
259
|
+
Author: Eric Wong <normalperson@yhbt.net>
|
260
|
+
Date: Fri Mar 18 14:23:51 2011 -0700
|
261
|
+
|
262
|
+
doc: various rdoc updates
|
263
|
+
|
264
|
+
Forcing myself to relearn my own library
|
265
|
+
|
266
|
+
commit cfa6c6b329ae44ae4f30a590d45e5df4c2d14d9a
|
267
|
+
Author: Eric Wong <normalperson@yhbt.net>
|
268
|
+
Date: Thu Mar 17 14:44:06 2011 -0700
|
269
|
+
|
270
|
+
tests: cleanup shadow warnings under 1.9.2
|
271
|
+
|
272
|
+
commit 1216ec9fc829c8c65f0f8ae2376138b90a1083d3
|
273
|
+
Author: Eric Wong <normalperson@yhbt.net>
|
274
|
+
Date: Thu Mar 17 21:19:54 2011 +0000
|
275
|
+
|
276
|
+
pkg.mk: new task for checking Ruby warnings
|
277
|
+
|
278
|
+
commit efcb08f789d60ebec26eadaada9af09593fedb9b
|
279
|
+
Author: Eric Wong <normalperson@yhbt.net>
|
280
|
+
Date: Wed Mar 16 01:11:05 2011 +0000
|
281
|
+
|
282
|
+
remove needless assignments
|
283
|
+
|
284
|
+
More noise
|
285
|
+
|
286
|
+
commit 6ce8e006a94a852ed76ffeababa1cac2ed7889c0
|
287
|
+
Author: Eric Wong <normalperson@yhbt.net>
|
288
|
+
Date: Wed Mar 16 01:06:22 2011 +0000
|
289
|
+
|
290
|
+
content_md5: improve readability
|
291
|
+
|
292
|
+
commit 61c1164c39886a198d33697c14c9abf099e3e08f
|
293
|
+
Author: Eric Wong <normalperson@yhbt.net>
|
294
|
+
Date: Wed Mar 16 00:31:25 2011 +0000
|
295
|
+
|
296
|
+
use bundler
|
297
|
+
|
298
|
+
commit 2318ee6a732f71d38a7749c0663bf780135be49c
|
299
|
+
Author: Eric Wong <normalperson@yhbt.net>
|
300
|
+
Date: Mon Feb 28 04:27:26 2011 +0000
|
301
|
+
|
302
|
+
gemspec: bump kcar version
|
303
|
+
|
304
|
+
Newer is better
|
305
|
+
|
3
306
|
commit 32223af63dc5296cd09f2472ad60263d98f379a2
|
4
307
|
Author: Eric Wong <normalperson@yhbt.net>
|
5
308
|
Date: Thu Feb 24 08:28:34 2011 +0000
|
data/GIT-VERSION-FILE
CHANGED
@@ -1 +1 @@
|
|
1
|
-
GIT_VERSION = 0.
|
1
|
+
GIT_VERSION = 0.2.0
|
data/GIT-VERSION-GEN
CHANGED
data/GNUmakefile
CHANGED
@@ -2,4 +2,15 @@ all::
|
|
2
2
|
RSYNC_DEST := bogomips.org:/srv/bogomips/http_spew
|
3
3
|
rfproject := rainbows
|
4
4
|
rfpackage := http_spew
|
5
|
+
|
6
|
+
RUBY_VERSION_FILE = lib/http_spew/version.rb
|
7
|
+
pkg_extra += $(RUBY_VERSION_FILE)
|
5
8
|
include pkg.mk
|
9
|
+
|
10
|
+
$(RUBY_VERSION_FILE): GIT-VERSION-FILE
|
11
|
+
@$(RM) -f $@+
|
12
|
+
@echo >> $@+ '# -*- encoding: binary -*-'
|
13
|
+
@echo >> $@+ 'HTTP_Spew.const_set :VERSION, "$(GIT_VERSION)"'
|
14
|
+
@mv $@+ $@
|
15
|
+
|
16
|
+
build: $(RUBY_VERSION_FILE)
|
data/Gemfile
ADDED
data/LATEST
CHANGED
@@ -1,4 +1,7 @@
|
|
1
|
-
===
|
1
|
+
=== http_spew 0.2.0 - bugfixes and improvements / 2011-05-10 22:19 UTC
|
2
2
|
|
3
|
-
|
3
|
+
InputSpray may now be layered on top of ContentMD5 successfully.
|
4
|
+
Improved multi-threading abilities and tests.
|
5
|
+
|
6
|
+
Do not consider the API remotely stable, yet.
|
4
7
|
|
data/NEWS
CHANGED
@@ -1,3 +1,10 @@
|
|
1
|
+
=== http_spew 0.2.0 - bugfixes and improvements / 2011-05-10 22:19 UTC
|
2
|
+
|
3
|
+
InputSpray may now be layered on top of ContentMD5 successfully.
|
4
|
+
Improved multi-threading abilities and tests.
|
5
|
+
|
6
|
+
Do not consider the API remotely stable, yet.
|
7
|
+
|
1
8
|
=== HTTP spew 0.1.0 - initial release / 2011-02-24 08:50 UTC
|
2
9
|
|
3
10
|
Might as well...
|
data/http_spew.gemspec
CHANGED
@@ -18,7 +18,7 @@ Gem::Specification.new do |s|
|
|
18
18
|
s.rdoc_options = rdoc_options
|
19
19
|
s.rubyforge_project = %q{rainbows}
|
20
20
|
s.test_files = Dir["test/test_*.rb"]
|
21
|
-
s.add_dependency(%q<kcar>, "~> 0.
|
22
|
-
s.add_dependency(%q<kgio>, "~> 2.
|
21
|
+
s.add_dependency(%q<kcar>, "~> 0.2")
|
22
|
+
s.add_dependency(%q<kgio>, "~> 2.4")
|
23
23
|
s.add_development_dependency(%q<wrongdoc>, "~> 1.5")
|
24
24
|
end
|
@@ -1,12 +1,26 @@
|
|
1
1
|
# -*- encoding: binary -*-
|
2
2
|
#
|
3
|
-
#
|
3
|
+
# This is a OS-level pipe that overrides IO#read to provide
|
4
|
+
# IO#readpartial-like semantics while remaining Rack::Lint-compatible
|
5
|
+
# for EOF, meaning we return nil on EOF instead of raising EOFError.
|
4
6
|
class HTTP_Spew::ChunkyPipe < Kgio::Pipe
|
7
|
+
|
8
|
+
# other threads may force an error to be raised in the +read+
|
9
|
+
# method
|
5
10
|
attr_accessor :error
|
6
11
|
|
7
|
-
#
|
12
|
+
# Override IO#read to behave like IO#readpartial, but still return +nil+
|
13
|
+
# on EOF instead of raising EOFError.
|
8
14
|
def read(*args)
|
9
|
-
|
10
|
-
kgio_read(*args)
|
15
|
+
check_err!
|
16
|
+
kgio_read(*args) || check_err! || close
|
17
|
+
end
|
18
|
+
|
19
|
+
def check_err!
|
20
|
+
if defined?(@error)
|
21
|
+
closed? or close
|
22
|
+
raise @error
|
23
|
+
end
|
24
|
+
nil
|
11
25
|
end
|
12
26
|
end
|
@@ -0,0 +1,125 @@
|
|
1
|
+
require "thread"
|
2
|
+
require "io/wait"
|
3
|
+
|
4
|
+
module HTTP_Spew::ClassMethods
|
5
|
+
def error_all(requests, error) # :nodoc:
|
6
|
+
requests.each { |req| req.error ||= error }
|
7
|
+
end
|
8
|
+
|
9
|
+
def done_early(ready, failed, requests) # :nodoc:
|
10
|
+
ready.concat(failed)
|
11
|
+
pending = requests - ready
|
12
|
+
unless pending.empty?
|
13
|
+
error = HTTP_Spew::ConnectionReset.new("prematurely terminated")
|
14
|
+
ready.concat(error_all(pending, error))
|
15
|
+
end
|
16
|
+
ready.uniq!
|
17
|
+
ready
|
18
|
+
end
|
19
|
+
|
20
|
+
# Returns an array of requests that are complete, including those
|
21
|
+
# that have errored out. Incomplete requests remain in +requests+
|
22
|
+
# If +need+ is fullfilled, it closes all incomplete requests and
|
23
|
+
# returns all requests.
|
24
|
+
def wait_nonblock!(need, requests)
|
25
|
+
ready, failed = [], []
|
26
|
+
requests.delete_if do |req|
|
27
|
+
begin
|
28
|
+
case req.resume
|
29
|
+
when Symbol # :wait_writable, :wait_readable
|
30
|
+
false
|
31
|
+
else
|
32
|
+
(ready << req).size == need and
|
33
|
+
return done_early(ready, failed, requests)
|
34
|
+
true
|
35
|
+
end
|
36
|
+
rescue => e
|
37
|
+
req.error = e
|
38
|
+
failed << req
|
39
|
+
end
|
40
|
+
end
|
41
|
+
ready.concat(failed).empty? ? nil : ready
|
42
|
+
end
|
43
|
+
|
44
|
+
def with_timeout(t)
|
45
|
+
t0 = Time.now
|
46
|
+
yield
|
47
|
+
ensure
|
48
|
+
t[0] -= Time.now - t0
|
49
|
+
t[0] = 0.0 if t[0] < 0
|
50
|
+
end
|
51
|
+
|
52
|
+
# Returns an array of requests that are complete, including those
|
53
|
+
# that have errored out.
|
54
|
+
# If +need+ is fullfilled, it closes all incomplete requests.
|
55
|
+
def wait_mt(need, requests, timeout)
|
56
|
+
ready, failed = [], []
|
57
|
+
r, w = Kgio::Pipe.new
|
58
|
+
active = []
|
59
|
+
t = [ timeout ]
|
60
|
+
requests.each_with_index do |req, i|
|
61
|
+
active << Thread.new do
|
62
|
+
begin
|
63
|
+
rv = req.run(timeout)
|
64
|
+
w.write([ i ].pack("v"))
|
65
|
+
rv
|
66
|
+
rescue => err
|
67
|
+
err
|
68
|
+
end
|
69
|
+
end
|
70
|
+
end
|
71
|
+
begin
|
72
|
+
with_timeout(t) { r.wait(t[0]) }
|
73
|
+
req_idx = r.read(2).unpack("v")[0]
|
74
|
+
thr = active[req_idx]
|
75
|
+
with_timeout(t) { thr.join(t[0]) }
|
76
|
+
rv = thr.value
|
77
|
+
(Array === rv ? ready : failed) << requests[req_idx]
|
78
|
+
ready.size == need and return done_early(ready, failed, requests)
|
79
|
+
end until t[0] < 0.0 || (ready.size + failed.size) == requests.size
|
80
|
+
|
81
|
+
ready.concat(failed)
|
82
|
+
pending = requests - ready
|
83
|
+
error = HTTP_Spew::TimeoutError.new("request timed out")
|
84
|
+
ready.concat(error_all(pending, error))
|
85
|
+
ensure
|
86
|
+
w.close
|
87
|
+
r.close
|
88
|
+
end
|
89
|
+
|
90
|
+
def wait(need, requests, timeout)
|
91
|
+
ready, failed = [], []
|
92
|
+
pollset = {}
|
93
|
+
t = [ timeout ]
|
94
|
+
begin
|
95
|
+
requests.each do |req|
|
96
|
+
begin
|
97
|
+
case rv = req.resume
|
98
|
+
when Symbol # :wait_writable, :wait_readable
|
99
|
+
pollset[req] = rv
|
100
|
+
else
|
101
|
+
(ready << req).size == need and
|
102
|
+
return done_early(ready, failed, requests)
|
103
|
+
pollset.delete(req)
|
104
|
+
end
|
105
|
+
rescue => e
|
106
|
+
req.error = e
|
107
|
+
failed << req
|
108
|
+
pollset.delete(req)
|
109
|
+
end
|
110
|
+
end
|
111
|
+
break if pollset.empty?
|
112
|
+
|
113
|
+
busy = pollset.keys
|
114
|
+
rv = with_timeout(t) { Kgio.poll(pollset, (t[0] * 1000).to_i) } or break
|
115
|
+
end while t[0] > 0.0 && requests = rv.keys.concat(busy).uniq!
|
116
|
+
|
117
|
+
ready.concat(failed)
|
118
|
+
unless requests.empty?
|
119
|
+
error = HTTP_Spew::TimeoutError.new("request timed out")
|
120
|
+
ready.concat(error_all(requests, error))
|
121
|
+
ready.uniq!
|
122
|
+
end
|
123
|
+
ready
|
124
|
+
end
|
125
|
+
end
|
@@ -1,52 +1,62 @@
|
|
1
1
|
# -*- encoding: binary -*-
|
2
2
|
require "digest/md5"
|
3
|
-
module HTTP_Spew::ContentMD5
|
4
|
-
class MismatchError < HTTP_Spew::Error
|
5
|
-
end
|
6
|
-
class LengthError < HTTP_Spew::Error
|
7
|
-
end
|
8
3
|
|
9
|
-
|
4
|
+
# this uses a pipe internally so it can use IO.tee in the "io_splice" RubyGem
|
5
|
+
class HTTP_Spew::ContentMD5
|
6
|
+
attr_reader :to_io
|
7
|
+
attr_reader :content_md5
|
8
|
+
attr_reader :bytes_digested
|
9
|
+
|
10
|
+
CRLF = "\r\n" # :nodoc:
|
11
|
+
|
12
|
+
def initialize(env)
|
10
13
|
if trailer = env["HTTP_TRAILER"]
|
11
|
-
|
12
|
-
return env["rack.input"] if have_md5 && env["HTTP_CONTENT_MD5"]
|
13
|
-
end
|
14
|
-
if trailer
|
15
|
-
unless have_md5
|
14
|
+
unless trailer.split(/\s*,\s*/).grep(/\AContent-MD5\z/i)[0]
|
16
15
|
trailer << (trailer.empty? ? "Content-MD5" : ",Content-MD5")
|
17
16
|
end
|
18
17
|
else
|
19
18
|
env["HTTP_TRAILER"] = "Content-MD5"
|
20
19
|
end
|
21
20
|
env["HTTP_TRANSFER_ENCODING"] = "chunked"
|
22
|
-
|
23
|
-
|
24
|
-
|
25
|
-
start_write_driver(env["rack.input"],
|
26
|
-
|
21
|
+
@to_io, wr = HTTP_Spew::ChunkyPipe.new
|
22
|
+
expect_md5 = env.delete("HTTP_CONTENT_MD5")
|
23
|
+
expect_len = env.delete("CONTENT_LENGTH")
|
24
|
+
start_write_driver(env["rack.input"], wr, expect_md5, expect_len)
|
25
|
+
end
|
26
|
+
|
27
|
+
# compatible with IO#read and Rack::InputWrapper#read
|
28
|
+
def read(length, buffer = "")
|
29
|
+
# calls HTTP_Spew::ChunkyPipe#read
|
30
|
+
@to_io.read(length, buffer)
|
27
31
|
end
|
28
32
|
|
29
|
-
def
|
33
|
+
def start_write_driver(input, wr, expect_md5, expect_len) # :nodoc:
|
30
34
|
Thread.new do
|
31
35
|
begin
|
32
|
-
digest
|
36
|
+
digest = Digest::MD5.new
|
37
|
+
buf = ""
|
38
|
+
bytes = 0
|
33
39
|
while input.read(0x4000, buf)
|
34
40
|
n = buf.size
|
35
41
|
bytes += n
|
36
|
-
wr.
|
42
|
+
wr.write("#{n.to_s(16)}\r\n")
|
37
43
|
digest.update(buf)
|
38
|
-
wr.
|
44
|
+
wr.write(buf << CRLF)
|
39
45
|
end
|
40
46
|
if expect_len && expect_len.to_i != bytes
|
41
|
-
raise LengthError,
|
47
|
+
raise HTTP_Spew::LengthError,
|
48
|
+
"expect=#{expect_len} != got=#{bytes}", []
|
42
49
|
end
|
43
|
-
|
44
|
-
if expect_md5 && expect_md5.strip !=
|
45
|
-
raise
|
50
|
+
md5 = [ digest.digest ].pack("m0")
|
51
|
+
if expect_md5 && expect_md5.strip != md5
|
52
|
+
raise HTTP_Spew::ChecksumError,
|
53
|
+
"expect=#{expect_md5} != got=#{md5}", []
|
46
54
|
end
|
47
|
-
wr.write "0\r\nContent-MD5: #{
|
55
|
+
wr.write "0\r\nContent-MD5: #{md5}\r\n\r\n"
|
56
|
+
@content_md5 = md5
|
57
|
+
@bytes_digested = bytes
|
48
58
|
rescue => e
|
49
|
-
|
59
|
+
@to_io.error = e
|
50
60
|
ensure
|
51
61
|
wr.close
|
52
62
|
end
|
data/lib/http_spew/headers.rb
CHANGED
@@ -10,12 +10,23 @@ module HTTP_Spew::Headers
|
|
10
10
|
CONTENT_TYPE = "CONTENT_TYPE" # specified by Rack to be !/^HTTP_/
|
11
11
|
# :startdoc:
|
12
12
|
|
13
|
+
# regenerates the request_uri from a Rack +env+
|
13
14
|
def request_uri(env)
|
14
15
|
qs = env[QUERY_STRING]
|
15
16
|
qs.size == 0 ? env[PATH_INFO] : "#{env[PATH_INFO]}?#{qs}"
|
16
17
|
end
|
17
18
|
module_function :request_uri
|
18
19
|
|
20
|
+
# converts a Rack +env+ into an HTTP header buffer
|
21
|
+
# If +input+ is a string, this appends +input+ to the header
|
22
|
+
# buffer so the client can avoid extra writes. If input is
|
23
|
+
# an IO-like object, it is returned as the second element
|
24
|
+
# of an array.
|
25
|
+
#
|
26
|
+
# This, the following code always works and may be used to clobber
|
27
|
+
# +input+ if it is merged into +buf+
|
28
|
+
#
|
29
|
+
# buf, input = env_to_headers(env, input)
|
19
30
|
def env_to_headers(env, input)
|
20
31
|
req = "#{env[REQUEST_METHOD]} " \
|
21
32
|
"#{env[REQUEST_URI] || request_uri(env)} HTTP/1.1\r\n" \
|