nyara 0.0.1.pre.2 → 0.0.1.pre.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 8eb89d789f199a49118e1f707db07aeaa8525421
-  data.tar.gz: 9e65ab8d0899ddad7baa96c78aff44fc26f9dd32
+  metadata.gz: 52583810c7afb74e920b0da7336e82441471ab82
+  data.tar.gz: 6fbd035f24bbd10ad1ade4ffdfce186e3a0d779f
 SHA512:
-  metadata.gz: 1ffc6507a00a079041e7c19eb7e44f37a29bc9f185a9c659cfb2a8449367567ffa87f82e90ce45cb63994b000a68214e86827714449d5e13d5b4d8b40415fa59
-  data.tar.gz: 7dd05d852cd406899a87eabbfb77b9a670981c783e3c4e5aa30850b03c415e1b0cbb17f77bbb4873e0d0b8c16c84db90d6f0d713bd63da415891061bd17bd093
+  metadata.gz: 7a3abf537a5027658497d171a5a593153316e60214b81c80f4f93c1e0557478f7473136b5cec6fc0daa76eedbb7650b01a284c72098a3069fa95bffd18ec468a
+  data.tar.gz: 8c5031033482180d9e0c1b3a4e541274abb562de5b1d04e81855c7ca556c7fcc44415abcf06a694b001f1c6a8dbd273509490a8ba45f3c58a0cf595921d92731

data/ext/accept.c

@@ -1,4 +1,4 @@
-/* efficient Accpet-* value parser in C */
+/* http accpet* value parser */
 
 #include "nyara.h"
 #include <ctype.h>
@@ -11,42 +11,39 @@
 // +level+ is a waste of time:
 //   http://stackoverflow.com/questions/13890996/http-accept-level
 
+#define ACCEPT_MAX 1000
+
 // sorted data structure
-typedef struct {
-  double* qs;
-  long len;
-  long cap;
-} QArray;
 
-static QArray qarray_new() {
-  QArray qa = {ALLOC_N(double, 10), 0, 10};
-  return qa;
-}
+static double qarray[ACCEPT_MAX];
+static long qarray_len = 0;
 
 // return inserted pos
-static long qarray_insert(QArray* qa, double v) {
-  if (qa->len == qa->cap) {
-    qa->cap *= 2;
-    REALLOC_N(qa->qs, double, qa->cap);
-  }
-  long i = 0;
-  for (; i < qa->len; i++) {
-    if (qa->qs[i] < v) {
-      memmove(qa->qs + i + 1, qa->qs + i, sizeof(double) * (qa->len - i));
+// the sort is "stable", which doesn't swap elements with the same q
+static long qarray_insert(double v) {
+  long i = qarray_len;
+  for (long j = qarray_len - 1; j >= 0; j--) {
+    if (qarray[j] < v) {
+      i = j;
+    } else {
       break;
     }
   }
-  qa->qs[i] = v;
-  qa->len++;
+  memmove(qarray + i + 1, qarray + i, sizeof(double) * (qarray_len - i));
+  qarray[i] = v;
+  qarray_len++;
   return i;
 }
 
-static void qarray_delete(QArray* qa) {
-  xfree(qa->qs);
-}
-
-static VALUE trim_space(volatile VALUE str) {
+// why truncate:
+// 1. normal user never send such long Aceept
+// 2. qarray_insert is O(n^2) in worst case, can lead to ddos vulnerability if there are more than 50000 accept entries
+static VALUE trim_space_and_truncate(volatile VALUE str) {
   long olen = RSTRING_LEN(str);
+  if (olen > ACCEPT_MAX) {
+    // todo log this exception
+    olen = ACCEPT_MAX;
+  }
   str = rb_str_new(RSTRING_PTR(str), olen);
   char* s = RSTRING_PTR(str);
   long len = 0;
@@ -80,8 +77,8 @@ static const char* find_q(const char* s, long len) {
   return NULL;
 }
 
-// parse a segment, and +out[value] = q+
-static void parse_seg(const char* s, long len, VALUE out, QArray* qa) {
+// parse a segment, and store in a sorted array, also updates qarray
+static void parse_seg(const char* s, long len, VALUE out) {
   double qval = 1;
   const char* q = find_q(s, len);
   if (q) {
@@ -97,7 +94,7 @@ static void parse_seg(const char* s, long len, VALUE out, QArray* qa) {
     }
     len = q - s;
   }
-  long pos = qarray_insert(qa, qval);
+  long pos = qarray_insert(qval);
   rb_ary_push(out, Qnil); // just to increase cap
   VALUE* out_ptr = RARRAY_PTR(out);
   long out_len = RARRAY_LEN(out); // note this len is +1
@@ -110,21 +107,20 @@ VALUE ext_parse_accept_value(VALUE _, volatile VALUE str) {
     return rb_ary_new();
   }
 
-  str = trim_space(str);
+  str = trim_space_and_truncate(str);
   const char* s = RSTRING_PTR(str);
   long len = RSTRING_LEN(str);
   volatile VALUE out = rb_ary_new();
-  QArray qa = qarray_new();
+  qarray_len = 0;
   while (len > 0) {
     long seg_len = find_seg(s, len);
     if (seg_len == 0) {
       break;
     }
-    parse_seg(s, seg_len, out, &qa);
+    parse_seg(s, seg_len, out);
     s += seg_len + 1;
     len -= seg_len + 1;
   }
-  qarray_delete(&qa);
   return out;
 }
 

data/ext/event.c

@@ -1,62 +1,167 @@
+/* unify API for epoll and kqueue */
+
 #include "nyara.h"
+#include "request.h"
 #include <sys/fcntl.h>
 #include <sys/socket.h>
 #include <errno.h>
 #include <stdint.h>
 #include <unistd.h>
 
-#define ETYPE_ACCEPT 0
-#define ETYPE_REQUEST 1
+#ifndef rb_obj_hide
+extern VALUE rb_obj_hide(VALUE obj);
+extern VALUE rb_obj_reveal(VALUE obj, VALUE klass);
+#endif
+
+#define ETYPE_CAN_ACCEPT 0
+#define ETYPE_HANDLE_REQUEST 1
 #define ETYPE_CONNECT 2
 #define MAX_E 1024
 static void loop_body(int fd, int etype);
 static int qfd;
 
+#define MAX_RECEIVE_DATA 65536
+static char received_data[MAX_RECEIVE_DATA];
+extern http_parser_settings nyara_request_parse_settings;
+
 #ifdef HAVE_KQUEUE
 #include "inc/kqueue.h"
 #elif HAVE_EPOLL
 #include "inc/epoll.h"
 #endif
 
-static void set_nonblock(int fd) {
+static VALUE fd_request_map;
+static VALUE watch_request_map;
+static ID id_not_found;
+static VALUE sym_term_close;
+static VALUE sym_writing;
+static VALUE sym_reading;
+static Request* curr_request;
+
+static void _set_nonblock(int fd) {
   int flags;
 
   if ((flags = fcntl(fd, F_GETFL)) == -1) {
-    rb_raise(rb_eRuntimeError, "fcntl(F_GETFL): %s", strerror(errno));
+    rb_sys_fail("fcntl(F_GETFL)");
   }
   if (fcntl(fd, F_SETFL, flags | O_NONBLOCK) == -1) {
-    rb_raise(rb_eRuntimeError, "fcntl(F_SETFL,O_NONBLOCK): %s", strerror(errno));
+    rb_sys_fail("fcntl(F_SETFL,O_NONBLOCK)");
   }
 }
 
-static VALUE ext_add(VALUE _, VALUE vfd) {
-  int fd = FIX2INT(vfd);
-  ADD_E(fd, ETYPE_CONNECT);
+static VALUE _fiber_func(VALUE _, VALUE args) {
+  VALUE instance = rb_ary_pop(args);
+  VALUE meth = rb_ary_pop(args);
+  rb_apply(instance, SYM2ID(meth), args);
   return Qnil;
 }
 
+static void _handle_request(VALUE request) {
+  Request* p;
+  Data_Get_Struct(request, Request, p);
+  curr_request = p;
+
+  // read and parse data
+  // NOTE we don't let http_parser invoke ruby code, because:
+  // 1. so the stack is shallower
+  // 2. Fiber.yield can pause http_parser, then the unparsed received_data is lost
+  long len = read(p->fd, received_data, MAX_RECEIVE_DATA);
+  if (len < 0) {
+    if (errno != EAGAIN && errno != EWOULDBLOCK) {
+      // this can happen when 2 events are fetched, and first event closes the fd, then second event fails
+      if (p->fd) {
+        nyara_detach_fd(p->fd);
+        p->fd = 0;
+      }
+      return;
+    }
+  } else if (len) {
+    // note: for http_parser, len = 0 means eof reached
+    //       but when in a fd-becomes-writable event it can also be 0
+    http_parser_execute(&(p->hparser), &nyara_request_parse_settings, received_data, len);
+  }
+
+  if (!p->parse_state) {
+    return;
+  }
+
+  // ensure action
+  if (p->fiber == Qnil) {
+    volatile RouteResult result = nyara_lookup_route(p->method, p->path, p->accept);
+    if (RTEST(result.controller)) {
+      rb_ary_push(result.args, rb_class_new_instance(1, &(p->self), result.controller));
+      // result.args is on stack, no need to worry gc
+      p->fiber = rb_fiber_new(_fiber_func, result.args);
+      p->scope = result.scope;
+      p->format = result.format;
+      p->response_header = rb_class_new_instance(0, NULL, nyara_header_hash_class);
+      p->response_header_extra_lines = rb_ary_new();
+    } else {
+      rb_funcall(p->self, id_not_found, 0);
+      nyara_detach_fd(p->fd);
+      p->fd = 0;
+      return;
+    }
+  }
+
+  // resume action
+  VALUE state = rb_fiber_resume(p->fiber, 0, NULL);
+  if (state == Qnil) { // _fiber_func always returns Qnil
+    // terminated (todo log raised error ?)
+    nyara_request_term_close(request, false);
+  } else if (state == sym_term_close) {
+    nyara_request_term_close(request, true);
+  } else if (state == sym_writing) {
+    // do nothing
+  } else if (state == sym_reading) {
+    // do nothing
+  } else {
+    // double value: sleep
+  }
+}
+
 // platform independent, invoked by LOOP_E()
 static void loop_body(int fd, int etype) {
   switch (etype) {
-    case ETYPE_ACCEPT: {
+    case ETYPE_CAN_ACCEPT: {
       int cfd = accept(fd, NULL, NULL);
       if (cfd > 0) {
-        set_nonblock(cfd);
-        ADD_E(cfd, ETYPE_REQUEST);
+        _set_nonblock(cfd);
+        ADD_E(cfd, ETYPE_HANDLE_REQUEST);
       }
       break;
     }
-    case ETYPE_REQUEST: {
-      nyara_handle_request(fd);
+    case ETYPE_HANDLE_REQUEST: {
+      VALUE key = INT2FIX(fd);
+      volatile VALUE request = rb_hash_aref(fd_request_map, key);
+      if (request == Qnil) {
+        request = nyara_request_new(fd);
+        rb_hash_aset(fd_request_map, key, request);
+      }
+      _handle_request(request);
       break;
     }
     case ETYPE_CONNECT: {
-      // todo
-      // NOTE
-      // fd and connection are 1:1, there can more more than 1 fds on a same file / address
-      // so it's streight forward to using fd as query index
+      VALUE request = rb_hash_aref(watch_request_map, INT2FIX(fd));
+      if (request != Qnil) {
+        _handle_request(request);
+      }
+    }
+  }
+}
+
+void nyara_detach_fd(int fd) {
+  VALUE request = rb_hash_delete(fd_request_map, INT2FIX(fd));
+  if (request != Qnil) {
+    Request* p;
+    Data_Get_Struct(request, Request, p);
+    VALUE* watched = RARRAY_PTR(p->watched_fds);
+    long watched_len = RARRAY_LEN(p->watched_fds);
+    for (long i = 0; i < watched_len; i++) {
+      rb_hash_delete(watch_request_map, watched[i]);
     }
   }
+  close(fd);
 }
 
 static VALUE ext_init_queue(VALUE _) {
@@ -66,8 +171,8 @@ static VALUE ext_init_queue(VALUE _) {
 
 static VALUE ext_run_queue(VALUE _, VALUE v_fd) {
   int fd = FIX2INT(v_fd);
-  set_nonblock(fd);
-  ADD_E(fd, ETYPE_ACCEPT);
+  _set_nonblock(fd);
+  ADD_E(fd, ETYPE_CAN_ACCEPT);
 
   LOOP_E();
   return Qnil;
@@ -75,15 +180,106 @@ static VALUE ext_run_queue(VALUE _, VALUE v_fd) {
 
 static VALUE ext_set_nonblock(VALUE _, VALUE v_fd) {
   int fd = FIX2INT(v_fd);
-  set_nonblock(fd);
+  _set_nonblock(fd);
   return Qnil;
 }
 
+static VALUE ext_fd_watch(VALUE _, VALUE v_fd) {
+  int fd = NUM2INT(v_fd);
+  rb_hash_aset(watch_request_map, v_fd, curr_request->self);
+  rb_ary_push(curr_request->watched_fds, v_fd);
+  ADD_E(fd, ETYPE_CONNECT);
+  return Qnil;
+}
+
+// override TCPSocket.send
+// returns sent length
+static VALUE ext_fd_send(VALUE _, VALUE v_fd, VALUE v_data, VALUE v_flags) {
+  int flags = NUM2INT(v_flags);
+  int fd = NUM2INT(v_fd);
+  char* buf = RSTRING_PTR(v_data);
+  long len = RSTRING_LEN(v_data);
+
+  // similar to _send_data in request.c
+  while(len) {
+    long written = send(fd, buf, len, flags);
+    if (written <= 0) {
+      if (errno == EAGAIN || errno == EWOULDBLOCK) {
+        rb_fiber_yield(1, &sym_writing);
+        continue;
+      } else {
+        rb_sys_fail("send(2)");
+        break;
+      }
+    } else {
+      buf += written;
+      len -= written;
+      if (len) {
+        rb_fiber_yield(1, &sym_writing);
+      }
+    }
+  }
+
+  return LONG2NUM(RSTRING_LEN(v_data) - len);
+}
+
+// override TCPSocket.recv
+// simulate blocking recv len or eof
+static VALUE ext_fd_recv(VALUE _, VALUE v_fd, VALUE v_len, VALUE v_flags) {
+  int flags = NUM2INT(v_flags);
+  int fd = NUM2INT(v_fd);
+  long buf_len = NUM2INT(v_len); // int shall be large enough...
+
+  volatile VALUE str = rb_tainted_str_new(0, buf_len);
+  volatile VALUE klass = RBASIC(str)->klass;
+  rb_obj_hide(str);
+
+  char* s = RSTRING_PTR(str);
+  while(buf_len) {
+    long recved = recv(fd, s, buf_len, flags);
+    if (recved < 0) {
+      if (errno == EAGAIN || errno == EWOULDBLOCK) {
+        rb_fiber_yield(1, &sym_reading);
+        continue;
+      } else {
+        rb_sys_fail("recv(2)");
+        break;
+      }
+    } else if (recved == 0) { // reached EOF
+      break;
+    }
+    s += recved;
+    buf_len -= recved;
+  }
+
+  if (RBASIC(str)->klass || RSTRING_LEN(str) != NUM2INT(v_len)) {
+    rb_raise(rb_eRuntimeError, "buffer string modified");
+  }
+  rb_obj_reveal(str, klass);
+  if (buf_len) {
+    rb_str_set_len(str, RSTRING_LEN(str) - buf_len);
+  }
+  rb_obj_taint(str);
+
+  return str;
+}
+
 void Init_event(VALUE ext) {
-  // rb_define_singleton_method(c, "add", add_q, 2);
+  fd_request_map = rb_hash_new();
+  rb_gc_register_mark_object(fd_request_map);
+  watch_request_map = rb_hash_new();
+  rb_gc_register_mark_object(watch_request_map);
+  id_not_found = rb_intern("not_found");
+  sym_term_close = ID2SYM(rb_intern("term_close"));
+  sym_writing = ID2SYM(rb_intern("writing"));
+  sym_reading = ID2SYM(rb_intern("reading"));
+
   rb_define_singleton_method(ext, "init_queue", ext_init_queue, 0);
   rb_define_singleton_method(ext, "run_queue", ext_run_queue, 1);
 
-  // for test
+  // fd operations
   rb_define_singleton_method(ext, "set_nonblock", ext_set_nonblock, 1);
+  rb_define_singleton_method(ext, "fd_watch", ext_fd_watch, 1);
+  rb_define_singleton_method(ext, "fd_send", ext_fd_send, 3);
+  rb_define_singleton_method(ext, "fd_recv", ext_fd_recv, 3);
 }

data/ext/extconf.rb

@@ -12,11 +12,11 @@ end
 def tweak_cflags
   mf_conf = RbConfig::MAKEFILE_CONFIG
   if mf_conf['CC'] =~ /clang/
-    # enable c++11. this can not be installed on $CPPFLAGS, wtf??
-    mf_conf['CXXFLAGS'] << ' -stdlib=libc++ -std=c++11'
+    # enable c++0x. this can not be installed on $CPPFLAGS, wtf??
+    mf_conf['CXXFLAGS'] << ' -stdlib=libc++ -std=c++0x'
     $CFLAGS << ' $(xflags)'
   else
-    mf_conf['CXXFLAGS'] << ' -std=c++11'
+    mf_conf['CXXFLAGS'] << ' -std=c++0x'
     $CFLAGS << ' -std=c99 -Wno-declaration-after-statement $(xflags)'
   end
 

data/ext/hashes.c

@@ -1,4 +1,5 @@
-// provide handy hash variants
+/* handy hash variants */
+
 #include "nyara.h"
 #include <ruby/st.h>
 

data/ext/http_parser.c

@@ -1 +1,7 @@
+/* to make mkmf happy */
+
+#ifndef HTTP_PARSER_STRICT
+#define HTTP_PARSER_STRICT 0
+#endif
+
 #include "http-parser/http_parser.c"

data/ext/inc/epoll.h

@@ -1,3 +1,5 @@
+/* epoll event adapter */
+
 #pragma once
 
 #include <sys/epoll.h>
@@ -6,7 +8,7 @@ static struct epoll_event qevents[MAX_E];
 
 static void ADD_E(int fd, uint64_t etype) {
   struct epoll_event e;
-  e.events = EPOLLIN;
+  e.events = EPOLLIN | EPOLLOUT;
   e.data.u64 = (etype << 32) | (uint64_t)fd;
 
   // todo timeout
@@ -18,9 +20,11 @@ static void ADD_E(int fd, uint64_t etype) {
 # endif
 }
 
+// either epoll or kqueue removes the event watch from queue when fd closed
+// seems this is not required in epoll?
 static void DEL_E(int fd) {
   struct epoll_event e;
-  e.events = EPOLLIN;
+  e.events = EPOLLIN | EPOLLOUT;
   e.data.ptr = NULL;
 
 # ifdef NDEBUG
@@ -45,13 +49,11 @@ static void LOOP_E() {
     int sz = epoll_wait(qfd, qevents, MAX_E, 100);
 
     for (int i = 0; i < sz; i++) {
-      switch (qevents[i].events) {
-        case EPOLLIN: {
-          int fd = (int)(qevents[i].data.u64 & 0xFFFFFFFF);
-          int etype = (int)(qevents[i].data.u64 >> 32);
-          loop_body(fd, etype);
-          break;
-        }
+      if (qevents[i].events & (EPOLLIN | EPOLLOUT)) {
+        int fd = (int)(qevents[i].data.u64 & 0xFFFFFFFF);
+        int etype = (int)(qevents[i].data.u64 >> 32);
+        loop_body(fd, etype);
+        break;
       }
     }
     // execute other thread / interrupts

data/ext/inc/kqueue.h

@@ -1,3 +1,5 @@
+/* kqueue event adapter */
+
 #pragma once
 
 #include <sys/types.h>
@@ -8,15 +10,13 @@
 # include <sys/queue.h>
 #endif
 
-static void loop_body(int fd, int etype);
-
 #define MAX_E 1024
 static int qfd;
 static struct kevent qevents[MAX_E];
 
 static void ADD_E(int fd, uint64_t etype) {
   struct kevent e;
-  EV_SET(&e, fd, EVFILT_READ, EV_ADD, 0, 0, (void*)etype);
+  EV_SET(&e, fd, EVFILT_READ | EVFILT_WRITE, EV_ADD, 0, 0, (void*)etype);
   // todo timeout
 # ifdef NDEBUG
   kevent(qfd, &e, 1, NULL, 0, NULL);
@@ -26,9 +26,9 @@ static void ADD_E(int fd, uint64_t etype) {
 # endif
 }
 
-static void DEL_E(int fd) {
+static void DEL_E(int fd, int filter) {
   struct kevent e;
-  EV_SET(&e, fd, EVFILT_READ, EV_DELETE, 0, 0, NULL);
+  EV_SET(&e, fd, filter, EV_DELETE, 0, 0, NULL);
 # ifdef NDEBUG
   kevent(qfd, &e, 1, NULL, 0, NULL);
 # else
@@ -57,18 +57,15 @@ static void LOOP_E() {
     int sz = kevent(qfd, NULL, 0, qevents, MAX_E, &ts);
 
     for (int i = 0; i < sz; i++) {
-      switch (qevents[i].filter) {
-        case EVFILT_READ: {
-          int fd = (int)qevents[i].ident;
-          // EV_EOF is set if the read side of the socket is shutdown
-          // the event can keep flipping back to consume cpu if we don't remove it
-          if ((qevents[i].flags & EV_EOF)) {
-            DEL_E(fd);
-          } else {
-            loop_body(fd, (int)qevents[i].udata);
-          }
-          break;
-        }
+      int fd = (int)qevents[i].ident;
+      if (qevents[i].flags & EV_EOF) {
+        // EV_EOF is set if the read side of the socket is shutdown
+        // the event can keep flipping back to consume cpu if we don't remove it
+        DEL_E(fd, qevents[i].filter);
+      }
+      if (qevents[i].filter & (EVFILT_READ | EVFILT_WRITE)) {
+        loop_body(fd, (int)qevents[i].udata);
+        break;
       }
     }
     // execute other thread / interrupts

data/ext/inc/str_intern.h

@@ -1,6 +1,6 @@
-#pragma once
+/* string internals, from ruby source code string.c */
 
-// from string.c
+#pragma once
 
 #define RUBY_MAX_CHAR_LEN 16
 #define STR_TMPLOCK FL_USER7

data/ext/mime.c

@@ -1,3 +1,5 @@
+/* content-type matching */
+
 #include "nyara.h"
 #include <string.h>
 #include <ctype.h>

data/ext/multipart_parser.c

@@ -1 +1,3 @@
+/* to make mkmf happy */
+
 #include "multipart-parser-c/multipart_parser.c"

data/ext/nyara.c

@@ -1,3 +1,5 @@
+/* ext entrance */
+
 #include "nyara.h"
 #include <ruby/io.h>
 #include <sys/socket.h>

data/ext/nyara.h

@@ -9,35 +9,41 @@
 #endif
 
 
-/* -- event -- */
+/* event.c */
 void Init_event(VALUE ext);
+void nyara_detach_fd(int fd);
 
 
-/* -- request & response class -- */
+/* request_parse.c */
+void Init_request_parse(VALUE nyara);
+
+
+/* request.c */
 void Init_request(VALUE nyara, VALUE ext);
-void nyara_handle_request(int fd);
+VALUE nyara_request_new(int fd);
+void nyara_request_term_close(VALUE request, bool write_last_chunk);
 
 
-/* -- url encoded parse -- */
+/* url_encoded.c */
 void Init_url_encoded(VALUE ext);
 long nyara_parse_path(VALUE path, const char*s, long len);
 void nyara_parse_param(VALUE output, const char* s, long len);
 
 
-/* -- accept parse -- */
+/* accept.c */
 void Init_accept(VALUE ext);
 VALUE ext_parse_accept_value(VALUE _, VALUE str);
 
 
-/* -- mime parse and match -- */
+/* mime.c */
 void Init_mime(VALUE ext);
 VALUE ext_mime_match(VALUE _, VALUE request_accept, VALUE accept_mimes);
 
 
-/* -- hashes -- */
+/* hashes.c */
 void Init_hashes(VALUE nyara);
 
-// ab-cd => Ab-Cd
+// "ab-cd" => "Ab-Cd"
 // note str must be string created by nyara code
 void nyara_headerlize(VALUE str);
 int nyara_rb_hash_has_key(VALUE hash, VALUE key);
@@ -47,7 +53,7 @@ extern VALUE nyara_header_hash_class;
 extern VALUE nyara_config_hash_class;
 
 
-/* -- route -- */
+/* route.c */
 typedef struct {
   VALUE controller;
   VALUE args;