sensu-em 2.4.1-java → 2.5.0.beta-java

Sign up to get free protection for your applications and to get access to all the features.
Files changed (51) hide show
  1. checksums.yaml +4 -4
  2. data/.travis.yml +15 -5
  3. data/CHANGELOG.md +41 -1
  4. data/README.md +2 -3
  5. data/eventmachine.gemspec +2 -1
  6. data/ext/cmain.cpp +19 -3
  7. data/ext/ed.cpp +22 -8
  8. data/ext/em.cpp +123 -76
  9. data/ext/em.h +40 -6
  10. data/ext/eventmachine.h +2 -0
  11. data/ext/extconf.rb +16 -2
  12. data/ext/fastfilereader/extconf.rb +3 -0
  13. data/ext/fastfilereader/mapper.cpp +1 -1
  14. data/ext/project.h +11 -7
  15. data/ext/rubymain.cpp +38 -2
  16. data/ext/ssl.cpp +4 -1
  17. data/ext/ssl.h +4 -0
  18. data/java/src/com/rubyeventmachine/EventableChannel.java +8 -1
  19. data/lib/em/buftok.rb +34 -85
  20. data/lib/em/protocols/httpclient.rb +31 -11
  21. data/lib/em/protocols/line_and_text.rb +2 -3
  22. data/lib/em/protocols/linetext2.rb +0 -1
  23. data/lib/em/protocols/smtpserver.rb +32 -9
  24. data/lib/em/pure_ruby.rb +2 -2
  25. data/lib/em/tick_loop.rb +19 -19
  26. data/lib/em/version.rb +1 -1
  27. data/lib/eventmachine.rb +12 -4
  28. data/lib/jeventmachine.rb +22 -6
  29. data/lib/rubyeventmachine.jar +0 -0
  30. data/rakelib/package.rake +1 -1
  31. data/tests/em_test_helper.rb +4 -0
  32. data/tests/test_attach.rb +1 -0
  33. data/tests/test_basic.rb +14 -16
  34. data/tests/test_completion.rb +1 -0
  35. data/tests/test_connection_count.rb +1 -0
  36. data/tests/test_connection_write.rb +35 -0
  37. data/tests/test_epoll.rb +11 -14
  38. data/tests/test_httpclient.rb +43 -0
  39. data/tests/test_iterator.rb +6 -6
  40. data/tests/test_kb.rb +19 -25
  41. data/tests/test_many_fds.rb +22 -0
  42. data/tests/test_pause.rb +7 -2
  43. data/tests/test_pool.rb +2 -0
  44. data/tests/test_process_watch.rb +2 -0
  45. data/tests/test_processes.rb +7 -7
  46. data/tests/test_resolver.rb +33 -7
  47. data/tests/test_ssl_methods.rb +3 -4
  48. data/tests/test_ssl_verify.rb +62 -62
  49. data/tests/test_threaded_resource.rb +8 -0
  50. data/tmp/java/rubyeventmachine/.build +0 -0
  51. metadata +27 -10
data/ext/em.h CHANGED
@@ -22,10 +22,10 @@ See the file COPYING for complete licensing information.
22
22
 
23
23
  #ifdef BUILD_FOR_RUBY
24
24
  #include <ruby.h>
25
-
26
25
  #ifdef HAVE_RB_THREAD_FD_SELECT
27
26
  #define EmSelect rb_thread_fd_select
28
27
  #else
28
+ // ruby 1.9.1 and below
29
29
  #define EmSelect rb_thread_select
30
30
  #endif
31
31
 
@@ -69,9 +69,33 @@ See the file COPYING for complete licensing information.
69
69
  #define EmSelect select
70
70
  #endif
71
71
 
72
+ #if !defined(HAVE_RB_FDSET_T)
73
+ #define fd_check(n) (((n) < FD_SETSIZE) ? 1 : 0*fprintf(stderr, "fd %d too large for select\n", (n)))
74
+ // These definitions are cribbed from include/ruby/intern.h in Ruby 1.9.3,
75
+ // with this change: any macros that read or write the nth element of an
76
+ // fdset first call fd_check to make sure n is in bounds.
77
+ typedef fd_set rb_fdset_t;
78
+ #define rb_fd_zero(f) FD_ZERO(f)
79
+ #define rb_fd_set(n, f) do { if (fd_check(n)) FD_SET((n), (f)); } while(0)
80
+ #define rb_fd_clr(n, f) do { if (fd_check(n)) FD_CLR((n), (f)); } while(0)
81
+ #define rb_fd_isset(n, f) (fd_check(n) ? FD_ISSET((n), (f)) : 0)
82
+ #define rb_fd_copy(d, s, n) (*(d) = *(s))
83
+ #define rb_fd_dup(d, s) (*(d) = *(s))
84
+ #define rb_fd_resize(n, f) ((void)(f))
85
+ #define rb_fd_ptr(f) (f)
86
+ #define rb_fd_init(f) FD_ZERO(f)
87
+ #define rb_fd_init_copy(d, s) (*(d) = *(s))
88
+ #define rb_fd_term(f) ((void)(f))
89
+ #define rb_fd_max(f) FD_SETSIZE
90
+ #define rb_fd_select(n, rfds, wfds, efds, timeout) \
91
+ select(fd_check((n)-1) ? (n) : FD_SETSIZE, (rfds), (wfds), (efds), (timeout))
92
+ #define rb_thread_fd_select(n, rfds, wfds, efds, timeout) \
93
+ rb_thread_select(fd_check((n)-1) ? (n) : FD_SETSIZE, (rfds), (wfds), (efds), (timeout))
94
+ #endif
95
+
72
96
  class EventableDescriptor;
73
97
  class InotifyDescriptor;
74
-
98
+ struct SelectData_t;
75
99
 
76
100
  /********************
77
101
  class EventMachine_t
@@ -83,6 +107,9 @@ class EventMachine_t
83
107
  static int GetMaxTimerCount();
84
108
  static void SetMaxTimerCount (int);
85
109
 
110
+ static int GetSimultaneousAcceptCount();
111
+ static void SetSimultaneousAcceptCount (int);
112
+
86
113
  public:
87
114
  EventMachine_t (EMCallback);
88
115
  virtual ~EventMachine_t();
@@ -175,7 +202,7 @@ class EventMachine_t
175
202
  public:
176
203
  void _ReadLoopBreaker();
177
204
  void _ReadInotifyEvents();
178
- int NumCloseScheduled;
205
+ int NumCloseScheduled;
179
206
 
180
207
  private:
181
208
  enum {
@@ -213,8 +240,13 @@ class EventMachine_t
213
240
  unsigned LastTickCount;
214
241
  #endif
215
242
 
243
+ #ifdef OS_DARWIN
244
+ mach_timebase_info_data_t mach_timebase;
245
+ #endif
246
+
216
247
  private:
217
248
  bool bTerminateSignalReceived;
249
+ SelectData_t *SelectData;
218
250
 
219
251
  bool bEpoll;
220
252
  int epfd; // Epoll file-descriptor
@@ -239,13 +271,15 @@ struct SelectData_t
239
271
  struct SelectData_t
240
272
  {
241
273
  SelectData_t();
274
+ ~SelectData_t();
242
275
 
243
276
  int _Select();
277
+ void _Clear();
244
278
 
245
279
  int maxsocket;
246
- fd_set fdreads;
247
- fd_set fdwrites;
248
- fd_set fderrors;
280
+ rb_fdset_t fdreads;
281
+ rb_fdset_t fdwrites;
282
+ rb_fdset_t fderrors;
249
283
  timeval tv;
250
284
  int nSockets;
251
285
  };
data/ext/eventmachine.h CHANGED
@@ -96,6 +96,8 @@ extern "C" {
96
96
  void evma_set_timer_quantum (int);
97
97
  int evma_get_max_timer_count();
98
98
  void evma_set_max_timer_count (int);
99
+ int evma_get_simultaneous_accept_count();
100
+ void evma_set_simultaneous_accept_count (int);
99
101
  void evma_setuid_string (const char *username);
100
102
  void evma_stop_machine();
101
103
  float evma_get_heartbeat_interval();
data/ext/extconf.rb CHANGED
@@ -39,10 +39,13 @@ def manual_ssl_config
39
39
  check_libs(libs) and check_heads(heads)
40
40
  end
41
41
 
42
+ # Eager check devs tools
43
+ have_devel? if respond_to?(:have_devel?)
44
+
42
45
  if ENV['CROSS_COMPILING']
43
- openssl_version = ENV.fetch("OPENSSL_VERSION", "1.0.0j")
46
+ openssl_version = ENV.fetch("OPENSSL_VERSION", "1.0.1i")
44
47
  openssl_dir = File.expand_path("~/.rake-compiler/builds/openssl-#{openssl_version}/")
45
- if File.exists?(openssl_dir)
48
+ if File.exist?(openssl_dir)
46
49
  FileUtils.mkdir_p Dir.pwd+"/openssl/"
47
50
  FileUtils.cp Dir[openssl_dir+"/include/openssl/*.h"], Dir.pwd+"/openssl/", :verbose => true
48
51
  FileUtils.cp Dir[openssl_dir+"/lib*.a"], Dir.pwd, :verbose => true
@@ -72,6 +75,7 @@ add_define "HAVE_INOTIFY" if inotify = have_func('inotify_init', 'sys/inotify.h'
72
75
  add_define "HAVE_OLD_INOTIFY" if !inotify && have_macro('__NR_inotify_init', 'sys/syscall.h')
73
76
  add_define 'HAVE_WRITEV' if have_func('writev', 'sys/uio.h')
74
77
  add_define 'HAVE_RB_THREAD_FD_SELECT' if have_func('rb_thread_fd_select')
78
+ add_define 'HAVE_RB_FDSET_T' if have_type('rb_fdset_t', 'ruby/intern.h')
75
79
 
76
80
  have_func('rb_wait_for_single_fd')
77
81
  have_func('rb_enable_interrupt')
@@ -139,6 +143,8 @@ when /openbsd/
139
143
  CONFIG['LDSHAREDXX'] = "$(CXX) -shared -lstdc++ -fPIC"
140
144
 
141
145
  when /darwin/
146
+ add_define 'OS_DARWIN'
147
+
142
148
  # on Unix we need a g++ link, not gcc.
143
149
  # Ff line contributed by Daniel Harple.
144
150
  CONFIG['LDSHARED'] = "$(CXX) " + CONFIG['LDSHARED'].split[1..-1].join(' ')
@@ -166,6 +172,14 @@ else
166
172
  CONFIG['LDSHARED'] = "$(CXX) -shared"
167
173
  end
168
174
 
175
+ # Platform-specific time functions
176
+ if have_func('clock_gettime')
177
+ # clock_gettime is POSIX, but the monotonic clocks are not
178
+ have_const('CLOCK_MONOTONIC_RAW', 'time.h') # Linux
179
+ have_const('CLOCK_MONOTONIC', 'time.h') # Linux, Solaris, BSDs
180
+ else
181
+ have_func('gethrtime') # Older Solaris and HP-UX
182
+ end
169
183
 
170
184
  # solaris c++ compiler doesn't have make_pair()
171
185
  TRY_LINK.sub!('$(CC)', '$(CXX)')
@@ -12,6 +12,9 @@ def add_define(name)
12
12
  $defs.push("-D#{name}")
13
13
  end
14
14
 
15
+ # Eager check devs tools
16
+ have_devel? if respond_to?(:have_devel?)
17
+
15
18
  add_define 'BUILD_FOR_RUBY'
16
19
 
17
20
  # Minor platform details between *nix and Windows:
@@ -195,7 +195,7 @@ void Mapper_t::Close()
195
195
  }
196
196
  if (hFile != INVALID_HANDLE_VALUE) {
197
197
  CloseHandle (hFile);
198
- hMapping = INVALID_HANDLE_VALUE;
198
+ hFile = INVALID_HANDLE_VALUE;
199
199
  }
200
200
  }
201
201
 
data/ext/project.h CHANGED
@@ -78,6 +78,11 @@ typedef int SOCKET;
78
78
  #endif
79
79
  #endif /* _AIX */
80
80
 
81
+ #ifdef OS_DARWIN
82
+ #include <mach/mach.h>
83
+ #include <mach/mach_time.h>
84
+ #endif /* OS_DARWIN */
85
+
81
86
  #endif /* OS_UNIX */
82
87
 
83
88
  #ifdef OS_WIN32
@@ -85,7 +90,9 @@ typedef int SOCKET;
85
90
  // 18Jun12: fd_setsize must be changed in the ruby binary (not in this extension). redefining it also causes segvs, see eventmachine/eventmachine#333
86
91
  //#define FD_SETSIZE 1024
87
92
 
93
+ // WIN32_LEAN_AND_MEAN excludes APIs such as Cryptography, DDE, RPC, Shell, and Windows Sockets.
88
94
  #define WIN32_LEAN_AND_MEAN
95
+
89
96
  #include <windows.h>
90
97
  #include <winsock2.h>
91
98
  #include <ws2tcpip.h>
@@ -93,13 +100,10 @@ typedef int SOCKET;
93
100
  #include <fcntl.h>
94
101
  #include <assert.h>
95
102
 
96
- #define close closesocket
97
-
98
- typedef int socklen_t;
99
- #ifndef _PID_T_
100
- #define _PID_T_
101
- typedef int pid_t;
102
- #endif /* _PID_T_ */
103
+ // Use the Win32 wrapper library that Ruby owns to be able to close sockets with the close() function
104
+ #define RUBY_EXPORT
105
+ #include <ruby/defines.h>
106
+ #include <ruby/win32.h>
103
107
  #endif /* OS_WIN32 */
104
108
 
105
109
  #if !defined(_MSC_VER) || _MSC_VER > 1500
data/ext/rubymain.cpp CHANGED
@@ -408,8 +408,22 @@ static VALUE t_get_subprocess_status (VALUE self, VALUE signature)
408
408
  if (evma_get_subprocess_status (NUM2ULONG (signature), &status)) {
409
409
  if (evma_get_subprocess_pid (NUM2ULONG (signature), &pid)) {
410
410
  proc_status = rb_obj_alloc(rb_cProcStatus);
411
+
412
+ /* MRI Ruby uses hidden instance vars */
411
413
  rb_iv_set(proc_status, "status", INT2FIX(status));
412
414
  rb_iv_set(proc_status, "pid", INT2FIX(pid));
415
+
416
+ #ifdef RUBINIUS
417
+ /* Rubinius uses standard instance vars */
418
+ rb_iv_set(proc_status, "@pid", INT2FIX(pid));
419
+ if (WIFEXITED(status)) {
420
+ rb_iv_set(proc_status, "@status", INT2FIX(WEXITSTATUS(status)));
421
+ } else if(WIFSIGNALED(status)) {
422
+ rb_iv_set(proc_status, "@termsig", INT2FIX(WTERMSIG(status)));
423
+ } else if(WIFSTOPPED(status)){
424
+ rb_iv_set(proc_status, "@stopsig", INT2FIX(WSTOPSIG(status)));
425
+ }
426
+ #endif
413
427
  }
414
428
  }
415
429
 
@@ -611,7 +625,7 @@ static VALUE t_set_sock_opt (VALUE self, VALUE signature, VALUE lev, VALUE optna
611
625
  int fd = evma_get_file_descriptor (NUM2ULONG (signature));
612
626
  int level = NUM2INT(lev), option = NUM2INT(optname);
613
627
  int i;
614
- void *v;
628
+ const void *v;
615
629
  socklen_t len;
616
630
 
617
631
  switch (TYPE(optval)) {
@@ -799,6 +813,21 @@ static VALUE t_set_max_timer_count (VALUE self, VALUE ct)
799
813
  return Qnil;
800
814
  }
801
815
 
816
+ /********************
817
+ t_get/set_simultaneous_accept_count
818
+ ********************/
819
+
820
+ static VALUE t_get_simultaneous_accept_count (VALUE self)
821
+ {
822
+ return INT2FIX (evma_get_simultaneous_accept_count());
823
+ }
824
+
825
+ static VALUE t_set_simultaneous_accept_count (VALUE self, VALUE ct)
826
+ {
827
+ evma_set_simultaneous_accept_count (FIX2INT (ct));
828
+ return Qnil;
829
+ }
830
+
802
831
  /***************
803
832
  t_setuid_string
804
833
  ***************/
@@ -833,7 +862,12 @@ static VALUE t_invoke_popen (VALUE self, VALUE cmd)
833
862
  }
834
863
  strings[len] = NULL;
835
864
 
836
- const unsigned long f = evma_popen (strings);
865
+ unsigned long f = 0;
866
+ try {
867
+ f = evma_popen (strings);
868
+ } catch (std::runtime_error e) {
869
+ f = 0; // raise exception below
870
+ }
837
871
  if (!f) {
838
872
  char *err = strerror (errno);
839
873
  char buf[100];
@@ -1271,6 +1305,8 @@ extern "C" void Init_rubyeventmachine()
1271
1305
  rb_define_module_function (EmModule, "set_timer_quantum", (VALUE(*)(...))t_set_timer_quantum, 1);
1272
1306
  rb_define_module_function (EmModule, "get_max_timer_count", (VALUE(*)(...))t_get_max_timer_count, 0);
1273
1307
  rb_define_module_function (EmModule, "set_max_timer_count", (VALUE(*)(...))t_set_max_timer_count, 1);
1308
+ rb_define_module_function (EmModule, "get_simultaneous_accept_count", (VALUE(*)(...))t_get_simultaneous_accept_count, 0);
1309
+ rb_define_module_function (EmModule, "set_simultaneous_accept_count", (VALUE(*)(...))t_set_simultaneous_accept_count, 1);
1274
1310
  rb_define_module_function (EmModule, "setuid_string", (VALUE(*)(...))t_setuid_string, 1);
1275
1311
  rb_define_module_function (EmModule, "invoke_popen", (VALUE(*)(...))t_invoke_popen, 1);
1276
1312
  rb_define_module_function (EmModule, "send_file_data", (VALUE(*)(...))t_send_file_data, 2);
data/ext/ssl.cpp CHANGED
@@ -400,13 +400,16 @@ int SslBox_t::PutPlaintext (const char *buf, int bufsize)
400
400
 
401
401
  bool fatal = false;
402
402
  bool did_work = false;
403
+ int pending = BIO_pending(pbioWrite);
403
404
 
404
- while (OutboundQ.HasPages()) {
405
+ while (OutboundQ.HasPages() && pending < SSLBOX_WRITE_BUFFER_SIZE) {
405
406
  const char *page;
406
407
  int length;
407
408
  OutboundQ.Front (&page, &length);
408
409
  assert (page && (length > 0));
409
410
  int n = SSL_write (pSSL, page, length);
411
+ pending = BIO_pending(pbioWrite);
412
+
410
413
  if (n > 0) {
411
414
  did_work = true;
412
415
  OutboundQ.PopFront();
data/ext/ssl.h CHANGED
@@ -54,6 +54,10 @@ class SslContext_t
54
54
  class SslBox_t
55
55
  **************/
56
56
 
57
+ #define SSLBOX_INPUT_CHUNKSIZE 2019
58
+ #define SSLBOX_OUTPUT_CHUNKSIZE 2048
59
+ #define SSLBOX_WRITE_BUFFER_SIZE 8192 // (SSLBOX_OUTPUT_CHUNKSIZE * 4)
60
+
57
61
  class SslBox_t
58
62
  {
59
63
  public:
@@ -73,7 +73,14 @@ public abstract class EventableChannel<OutboundPacketType> {
73
73
 
74
74
  public void setCommInactivityTimeout (long seconds) {
75
75
  // TODO
76
- System.out.println ("SET COMM INACTIVITY UNIMPLEMENTED IN JRUBY" + seconds);
76
+ /**
77
+ * A proper fix will require implementing a heartbeat in
78
+ * the java reactor similar to the one used for connect
79
+ * timeouts in the c++ reactor. You could also emulate
80
+ * it with timers in the jeventmachine.rb wrapper.
81
+ *
82
+ * https://github.com/igrigorik/em-http-request/issues/85
83
+ */
77
84
  }
78
85
 
79
86
  public abstract Object[] getPeerName();
data/lib/em/buftok.rb CHANGED
@@ -1,110 +1,59 @@
1
1
  # BufferedTokenizer takes a delimiter upon instantiation, or acts line-based
2
2
  # by default. It allows input to be spoon-fed from some outside source which
3
3
  # receives arbitrary length datagrams which may-or-may-not contain the token
4
- # by which entities are delimited.
5
- #
6
- # By default, new BufferedTokenizers will operate on lines delimited by "\n" by default
7
- # or allow you to specify any delimiter token you so choose, which will then
8
- # be used by String#split to tokenize the input data
9
- #
10
- # @example Using BufferedTokernizer to parse lines out of incoming data
11
- #
12
- # module LineBufferedConnection
13
- # def receive_data(data)
14
- # (@buffer ||= BufferedTokenizer.new).extract(data).each do |line|
15
- # receive_line(line)
16
- # end
17
- # end
18
- # end
19
- #
20
- # @author Tony Arcieri
21
- # @author Martin Emde
4
+ # by which entities are delimited. In this respect it's ideally paired with
5
+ # something like EventMachine (http://rubyeventmachine.com/).
22
6
  class BufferedTokenizer
23
- # @param [String] delimiter
24
- # @param [Integer] size_limit
25
- def initialize(delimiter = "\n", size_limit = nil)
26
- @delimiter = delimiter
27
- @size_limit = size_limit
28
-
29
- # The input buffer is stored as an array. This is by far the most efficient
30
- # approach given language constraints (in C a linked list would be a more
31
- # appropriate data structure). Segments of input data are stored in a list
32
- # which is only joined when a token is reached, substantially reducing the
33
- # number of objects required for the operation.
7
+ # New BufferedTokenizers will operate on lines delimited by a delimiter,
8
+ # which is by default the global input delimiter $/ ("\n").
9
+ #
10
+ # The input buffer is stored as an array. This is by far the most efficient
11
+ # approach given language constraints (in C a linked list would be a more
12
+ # appropriate data structure). Segments of input data are stored in a list
13
+ # which is only joined when a token is reached, substantially reducing the
14
+ # number of objects required for the operation.
15
+ def initialize(delimiter = $/)
16
+ @delimiter = delimiter
34
17
  @input = []
35
-
36
- # Size of the input buffer
37
- @input_size = 0
18
+ @tail = ''
19
+ @trim = @delimiter.length - 1
38
20
  end
39
21
 
40
22
  # Extract takes an arbitrary string of input data and returns an array of
41
- # tokenized entities, provided there were any available to extract.
42
- #
43
- # @example
23
+ # tokenized entities, provided there were any available to extract. This
24
+ # makes for easy processing of datagrams using a pattern like:
44
25
  #
45
- # tokenizer.extract(data).
46
- # map { |entity| Decode(entity) }.each { ... }
26
+ # tokenizer.extract(data).map { |entity| Decode(entity) }.each do ...
47
27
  #
48
- # @param [String] data
28
+ # Using -1 makes split to return "" if the token is at the end of
29
+ # the string, meaning the last element is the start of the next chunk.
49
30
  def extract(data)
50
- # Extract token-delimited entities from the input string with the split command.
51
- # There's a bit of craftiness here with the -1 parameter. Normally split would
52
- # behave no differently regardless of if the token lies at the very end of the
53
- # input buffer or not (i.e. a literal edge case) Specifying -1 forces split to
54
- # return "" in this case, meaning that the last entry in the list represents a
55
- # new segment of data where the token has not been encountered
56
- entities = data.split @delimiter, -1
57
-
58
- # Check to see if the buffer has exceeded capacity, if we're imposing a limit
59
- if @size_limit
60
- raise 'input buffer full' if @input_size + entities.first.size > @size_limit
61
- @input_size += entities.first.size
31
+ if @trim > 0
32
+ tail_end = @tail.slice!(-@trim, @trim) # returns nil if string is too short
33
+ data = tail_end + data if tail_end
62
34
  end
63
35
 
64
- # Move the first entry in the resulting array into the input buffer. It represents
65
- # the last segment of a token-delimited entity unless it's the only entry in the list.
66
- @input << entities.shift
67
-
68
- # If the resulting array from the split is empty, the token was not encountered
69
- # (not even at the end of the buffer). Since we've encountered no token-delimited
70
- # entities this go-around, return an empty array.
71
- return [] if entities.empty?
72
-
73
- # At this point, we've hit a token, or potentially multiple tokens. Now we can bring
74
- # together all the data we've buffered from earlier calls without hitting a token,
75
- # and add it to our list of discovered entities.
76
- entities.unshift @input.join
36
+ @input << @tail
37
+ entities = data.split(@delimiter, -1)
38
+ @tail = entities.shift
77
39
 
78
- # Now that we've hit a token, joined the input buffer and added it to the entities
79
- # list, we can go ahead and clear the input buffer. All of the segments that were
80
- # stored before the join can now be garbage collected.
81
- @input.clear
82
-
83
- # The last entity in the list is not token delimited, however, thanks to the -1
84
- # passed to split. It represents the beginning of a new list of as-yet-untokenized
85
- # data, so we add it to the start of the list.
86
- @input << entities.pop
87
-
88
- # Set the new input buffer size, provided we're keeping track
89
- @input_size = @input.first.size if @size_limit
40
+ unless entities.empty?
41
+ @input << @tail
42
+ entities.unshift @input.join
43
+ @input.clear
44
+ @tail = entities.pop
45
+ end
90
46
 
91
- # Now we're left with the list of extracted token-delimited entities we wanted
92
- # in the first place. Hooray!
93
47
  entities
94
48
  end
95
49
 
96
50
  # Flush the contents of the input buffer, i.e. return the input buffer even though
97
- # a token has not yet been encountered.
98
- #
99
- # @return [String]
51
+ # a token has not yet been encountered
100
52
  def flush
53
+ @input << @tail
101
54
  buffer = @input.join
102
55
  @input.clear
56
+ @tail = "" # @tail.clear is slightly faster, but not supported on 1.8.7
103
57
  buffer
104
58
  end
105
-
106
- # @return [Boolean]
107
- def empty?
108
- @input.empty?
109
- end
110
59
  end
@@ -23,8 +23,6 @@
23
23
  #
24
24
  #
25
25
 
26
-
27
-
28
26
  module EventMachine
29
27
  module Protocols
30
28
 
@@ -52,7 +50,6 @@ module EventMachine
52
50
  # DNS: Some way to cache DNS lookups for hostnames we connect to. Ruby's
53
51
  # DNS lookups are unbelievably slow.
54
52
  # HEAD requests.
55
- # Chunked transfer encoding.
56
53
  # Convenience methods for requests. get, post, url, etc.
57
54
  # SSL.
58
55
  # Handle status codes like 304, 100, etc.
@@ -184,6 +181,8 @@ module EventMachine
184
181
  @content_length = nil # not zero
185
182
  @content = ""
186
183
  @status = nil
184
+ @chunked = false
185
+ @chunk_length = nil
187
186
  @read_state = :header
188
187
  @connection_close = nil
189
188
  when :header
@@ -191,7 +190,7 @@ module EventMachine
191
190
  if ary.length == 2
192
191
  data = ary.last
193
192
  if ary.first == ""
194
- if (@content_length and @content_length > 0) || @connection_close
193
+ if (@content_length and @content_length > 0) || @chunked || @connection_close
195
194
  @read_state = :content
196
195
  else
197
196
  dispatch_response
@@ -211,6 +210,8 @@ module EventMachine
211
210
  @content_length ||= $'.to_i
212
211
  elsif ary.first =~ /\Aconnection:\s*close/i
213
212
  @connection_close = true
213
+ elsif ary.first =~ /\Atransfer-encoding:\s*chunked/i
214
+ @chunked = true
214
215
  end
215
216
  end
216
217
  else
@@ -218,12 +219,32 @@ module EventMachine
218
219
  data = ""
219
220
  end
220
221
  when :content
221
- # If there was no content-length header, we have to wait until the connection
222
- # closes. Everything we get until that point is content.
223
- # TODO: Must impose a content-size limit, and also must implement chunking.
224
- # Also, must support either temporary files for large content, or calling
225
- # a content-consumer block supplied by the user.
226
- if @content_length
222
+ if @chunked && @chunk_length
223
+ bytes_needed = @chunk_length - @chunk_read
224
+ new_data = data[0, bytes_needed]
225
+ @chunk_read += new_data.length
226
+ @content += new_data
227
+ data = data[bytes_needed..-1] || ""
228
+ if @chunk_length == @chunk_read && data[0,2] == "\r\n"
229
+ @chunk_length = nil
230
+ data = data[2..-1]
231
+ end
232
+ elsif @chunked
233
+ if (m = data.match(/\A(\S*)\r\n/m))
234
+ data = data[m[0].length..-1]
235
+ @chunk_length = m[1].to_i(16)
236
+ @chunk_read = 0
237
+ if @chunk_length == 0
238
+ dispatch_response
239
+ @read_state = :base
240
+ end
241
+ end
242
+ elsif @content_length
243
+ # If there was no content-length header, we have to wait until the connection
244
+ # closes. Everything we get until that point is content.
245
+ # TODO: Must impose a content-size limit, and also must implement chunking.
246
+ # Also, must support either temporary files for large content, or calling
247
+ # a content-consumer block supplied by the user.
227
248
  bytes_needed = @content_length - @content.length
228
249
  @content += data[0, bytes_needed]
229
250
  data = data[bytes_needed..-1] || ""
@@ -274,6 +295,5 @@ module EventMachine
274
295
  end
275
296
  end
276
297
  end
277
-
278
298
  end
279
299
  end
@@ -32,7 +32,6 @@ module EventMachine
32
32
  # for a version which is optimized for correctness with regard to binary text blocks
33
33
  # that can switch back to line mode.
34
34
  class LineAndTextProtocol < Connection
35
- MaxLineLength = 16*1024
36
35
  MaxBinaryLength = 32*1024*1024
37
36
 
38
37
  def initialize *args
@@ -42,7 +41,7 @@ module EventMachine
42
41
  def receive_data data
43
42
  if @lbp_mode == :lines
44
43
  begin
45
- @lpb_buffer.extract(data).each do |line|
44
+ @lpb_buffer.extract(data).each do |line|
46
45
  receive_line(line.chomp) if respond_to?(:receive_line)
47
46
  end
48
47
  rescue Exception
@@ -116,7 +115,7 @@ module EventMachine
116
115
  #--
117
116
  # For internal use, establish protocol baseline for handling lines.
118
117
  def lbp_init_line_state
119
- @lpb_buffer = BufferedTokenizer.new("\n", MaxLineLength)
118
+ @lpb_buffer = BufferedTokenizer.new("\n")
120
119
  @lbp_mode = :lines
121
120
  end
122
121
  private :lbp_init_line_state
@@ -37,7 +37,6 @@ module EventMachine
37
37
  # When we get around to that, call #receive_error if the user defined it, otherwise
38
38
  # throw exceptions.
39
39
 
40
- MaxLineLength = 16*1024
41
40
  MaxBinaryLength = 32*1024*1024
42
41
 
43
42
  #--