nyara 0.0.1.pre.8 → 0.0.1.pre.9

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 1464261ce5d4ec90868a68278e81de0124568897
-  data.tar.gz: 566342b6927ac0fc3f5607580a0863d18b9be432
+  metadata.gz: abcd7918ded6547e4edadb0746742a854f3c216b
+  data.tar.gz: 8dfaf81082c63b6e117bfc6836210d3d6f8f9aa8
 SHA512:
-  metadata.gz: c322de1bd85993df20041bd8535800751abdf4b4c8152b7ed4899a0430aed90d4552a97f21839267376f78bac18cc254fd395e9124cc071db8e4b3575ba6537c
-  data.tar.gz: e7c16bb717db1c131069d361087a8d97e96b1282d93d002582aa10cd0bcff01741eedefa0d43ab65c18c9ec480e1c214d3f03218c4569a1bb3786194c43c46e8
+  metadata.gz: 00c038fe5d84059b5452c8f445e0680c95f05a81be846c73a481efe0fd1a2d50433f3511b04453187983e6fd491940c7033229757d10d5da52c91fd508c332e8
+  data.tar.gz: 8a49ba878195450a06fe361c3d0ac57697c89f508b2c8f4ba5bb0e5a0acbcd8ff2f287cf59f38d63df4633af17c055b98ea1d6d02623318a81098b55a8b4b7b1

data/bin/nyara

@@ -0,0 +1,17 @@
+#!/usr/bin/env ruby
+require_relative "../lib/nyara/nyara"
+
+args = ARGV.dup
+task_name = args.shift || 'help'
+
+
+case task_name
+when 'help'
+  Nyara::Command.help
+when '-h'
+  Nyara::Command.help
+when '-v'
+  Nyara::Command.version
+when 'new'
+  Nyara::Command.new_project(*args)
+end

data/changes

@@ -0,0 +1,12 @@
+2013-07-13 add bin script `nyara`
+2013-07-10 add before filter
+2013-07-06 add worker signal management
+2013-07-05 add static file controller
+           Config: remove setter helpers, add path helers
+2013-07-03 add multipart request parsing
+2013-07-01 add flash helper
+2013-06-30 add Nyara::Test::Response for test to parse response
+2013-06-27 compatibility fixes for versions of GCC and Ruby
+2013-06-26 patch TCPSocket so client libraries in Ruby won't stuck
+           add sleep helper
+2013-06-22 add changes and copying

data/copying

@@ -0,0 +1,10 @@
+Copyright (c) 2013, Zete Lui
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met:
+
+- Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer.
+- Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution.
+- Neither the name of the *Nyara* nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

data/ext/event.c

@@ -7,23 +7,14 @@
 #include <errno.h>
 #include <stdint.h>
 #include <unistd.h>
+#include <ruby/st.h>
 
-#ifndef rb_obj_hide
-extern VALUE rb_obj_hide(VALUE obj);
-extern VALUE rb_obj_reveal(VALUE obj, VALUE klass);
-#endif
-
-#define ETYPE_CAN_ACCEPT 0
-#define ETYPE_HANDLE_REQUEST 1
-#define ETYPE_CONNECT 2
 #define MAX_E 1024
-static void loop_body(int fd, int etype);
+static void loop_body(VALUE rid);
 static void loop_check();
 static int qfd = 0;
-
-#define MAX_RECEIVE_DATA 65536 * 2
-static char received_data[MAX_RECEIVE_DATA];
-extern http_parser_settings nyara_request_parse_settings;
+static int tcp_server_fd = 0;
+static st_table* handled_rids; // {rid => nil} for current round
 
 #ifdef HAVE_KQUEUE
 #include "inc/kqueue.h"
@@ -31,15 +22,27 @@ extern http_parser_settings nyara_request_parse_settings;
 #include "inc/epoll.h"
 #endif
 
-static VALUE fd_request_map;
-static VALUE watch_request_map;
-static ID id_not_found;
+#ifndef rb_obj_hide
+extern VALUE rb_obj_hide(VALUE obj);
+extern VALUE rb_obj_reveal(VALUE obj, VALUE klass);
+#endif
+
+#define MAX_RECEIVE_DATA 65536 * 2
+static char received_data[MAX_RECEIVE_DATA];
+extern http_parser_settings nyara_request_parse_settings;
+
+static VALUE rid_request_map;    // {rid(FIXNUM) => request}
+static VALUE to_resume_requests; // [request], for current round
+
+static VALUE sym_accept;
+
+// Fiber.yield
 static VALUE sym_term_close;
 static VALUE sym_writing;
 static VALUE sym_reading;
 static VALUE sym_sleep;
+
 static Request* curr_request;
-static VALUE to_resume_actions;
 static bool graceful_quit = false;
 
 static VALUE _fiber_func(VALUE _, VALUE args) {
@@ -82,8 +85,7 @@ static void _handle_request(VALUE request) {
     if (errno != EAGAIN && errno != EWOULDBLOCK) {
       // this can happen when 2 events are fetched, and first event closes the fd, then second event fails
       if (p->fd) {
-        nyara_detach_fd(p->fd);
-        p->fd = 0;
+        nyara_detach_rid(p->rid);
       }
       return;
     }
@@ -100,6 +102,7 @@ static void _handle_request(VALUE request) {
   // ensure action
   if (p->fiber == Qnil) {
     volatile RouteResult result = nyara_lookup_route(p->method, p->path, p->accept);
+    nyara_summary_request(p->method, p->path, result.controller);
     if (RTEST(result.controller)) {
       rb_ary_push(result.args, rb_class_new_instance(1, &(p->self), result.controller));
       // result.args is on stack, no need to worry gc
@@ -117,8 +120,7 @@ static void _handle_request(VALUE request) {
         not_found_len = strlen(not_found);
       }
       nyara_send_data(p->fd, not_found, not_found_len);
-      nyara_detach_fd(p->fd);
-      p->fd = 0;
+      nyara_detach_rid(p->rid);
       return;
     }
   }
@@ -127,31 +129,26 @@ static void _handle_request(VALUE request) {
 }
 
 // platform independent, invoked by LOOP_E()
-static void loop_body(int fd, int etype) {
-  switch (etype) {
-    case ETYPE_CAN_ACCEPT: {
-      int cfd = accept(fd, NULL, NULL);
-      if (cfd > 0) {
-        nyara_set_nonblock(cfd);
-        ADD_E(cfd, ETYPE_HANDLE_REQUEST);
-      }
-      break;
+static void loop_body(VALUE rid) {
+  if (rid == sym_accept) {
+    int cfd = accept(tcp_server_fd, NULL, NULL);
+    if (cfd > 0) {
+      nyara_set_nonblock(cfd);
+      Request* p = nyara_request_new(cfd);
+      rb_hash_aset(rid_request_map, p->rid, p->self);
+      ADD_E(cfd, p->rid);
     }
-    case ETYPE_HANDLE_REQUEST: {
-      VALUE key = INT2FIX(fd);
-      volatile VALUE request = rb_hash_aref(fd_request_map, key);
-      if (request == Qnil) {
-        request = nyara_request_new(fd);
-        rb_hash_aset(fd_request_map, key, request);
-      }
-      _handle_request(request);
-      break;
+  } else {
+    // epoll_wait can return multiple results on a same request,
+    // and this request may be closed at previous round.
+    if (st_lookup(handled_rids, rid, NULL)) {
+      return;
     }
-    case ETYPE_CONNECT: {
-      VALUE request = rb_hash_aref(watch_request_map, INT2FIX(fd));
-      if (request != Qnil) {
-        _handle_request(request);
-      }
+    st_insert(handled_rids, rid, Qnil);
+
+    VALUE request = rb_hash_aref(rid_request_map, rid);
+    if (request != Qnil) {
+      _handle_request(request);
     }
   }
 }
@@ -161,52 +158,59 @@ static void loop_check() {
   rb_thread_schedule();
 
   // wakeup actions which finished sleeping
-  long len = RARRAY_LEN(to_resume_actions);
+  long len = RARRAY_LEN(to_resume_requests);
   if (len) {
-    VALUE* ptr = RARRAY_PTR(to_resume_actions);
+    VALUE* ptr = RARRAY_PTR(to_resume_requests);
     for (long i = 0; i < len; i++) {
       VALUE request = ptr[i];
       Request* p;
       Data_Get_Struct(request, Request, p);
 
       p->sleeping = false;
-      if (!rb_fiber_alive_p(p->fiber)) {
+      if (!rb_fiber_alive_p(p->fiber) || !p->fd) { // do not wake dead requests
         continue;
       }
 
       _resume_action(p);
       if (qfd) {
+        // printf("%s\n", "no way!");
+        // _Exit(1);
         VALUE* v_fds = RARRAY_PTR(p->watched_fds);
         long v_fds_len = RARRAY_LEN(p->watched_fds);
         for (long i = 0; i < v_fds_len; i++) {
-          ADD_E(FIX2INT(v_fds[i]), ETYPE_CONNECT);
+          ADD_E(FIX2INT(v_fds[i]), p->rid);
         }
-        ADD_E(p->fd, ETYPE_HANDLE_REQUEST);
+        ADD_E(p->fd, p->rid);
       } else {
         // we are in a test, no queue
       }
     }
+
+    rb_ary_clear(to_resume_requests);
   }
 
   if (graceful_quit) {
-    if (RTEST(rb_funcall(fd_request_map, rb_intern("empty?"), 0))) {
+    if (RTEST(rb_funcall(rid_request_map, rb_intern("empty?"), 0))) {
       _Exit(0);
     }
   }
 }
 
-void nyara_detach_fd(int fd) {
-  VALUE request = rb_hash_delete(fd_request_map, INT2FIX(fd));
+void nyara_detach_rid(VALUE rid) {
+  VALUE request = rb_hash_delete(rid_request_map, rid);
   if (request != Qnil) {
     Request* p;
     Data_Get_Struct(request, Request, p);
     VALUE* watched = RARRAY_PTR(p->watched_fds);
     long watched_len = RARRAY_LEN(p->watched_fds);
     for (long i = 0; i < watched_len; i++) {
-      rb_hash_delete(watch_request_map, watched[i]);
+      close(NUM2INT(watched[i]));
+    }
+    if (p->fd) {
+      close(p->fd);
+      p->fd = 0;
     }
   }
-  close(fd);
 }
 
 static VALUE ext_init_queue(VALUE _) {
@@ -216,9 +220,9 @@ static VALUE ext_init_queue(VALUE _) {
 
 // run queue loop with server_fd
 static VALUE ext_run_queue(VALUE _, VALUE v_server_fd) {
-  int fd = FIX2INT(v_server_fd);
-  nyara_set_nonblock(fd);
-  ADD_E(fd, ETYPE_CAN_ACCEPT);
+  tcp_server_fd = FIX2INT(v_server_fd);
+  nyara_set_nonblock(tcp_server_fd);
+  ADD_E(tcp_server_fd, sym_accept);
 
   LOOP_E();
   return Qnil;
@@ -255,7 +259,7 @@ static VALUE ext_request_sleep(VALUE _, VALUE request) {
 // NOTE this will be executed in another thread, resuming fiber in a non-main thread will stuck
 static VALUE ext_request_wakeup(VALUE _, VALUE request) {
   // NOTE should not use curr_request
-  rb_ary_push(to_resume_actions, request);
+  rb_ary_push(to_resume_requests, request);
   return Qnil;
 }
 
@@ -267,9 +271,15 @@ static VALUE ext_set_nonblock(VALUE _, VALUE v_fd) {
 
 static VALUE ext_fd_watch(VALUE _, VALUE v_fd) {
   int fd = NUM2INT(v_fd);
-  rb_hash_aset(watch_request_map, v_fd, curr_request->self);
   rb_ary_push(curr_request->watched_fds, v_fd);
-  ADD_E(fd, ETYPE_CONNECT);
+  ADD_E(fd, curr_request->rid);
+  return Qnil;
+}
+
+static VALUE ext_fd_unwatch(VALUE _, VALUE v_fd) {
+  int fd = NUM2INT(v_fd);
+  rb_ary_delete(curr_request->watched_fds, v_fd);
+  DEL_E(fd);
   return Qnil;
 }
 
@@ -367,19 +377,18 @@ static VALUE ext_handle_request(VALUE _, VALUE request) {
 }
 
 void Init_event(VALUE ext) {
-  fd_request_map = rb_hash_new();
-  rb_gc_register_mark_object(fd_request_map);
-  watch_request_map = rb_hash_new();
-  rb_gc_register_mark_object(watch_request_map);
-  id_not_found = rb_intern("not_found");
+  rid_request_map = rb_hash_new();
+  rb_gc_register_mark_object(rid_request_map);
+  handled_rids = st_init_numtable();
+  to_resume_requests = rb_ary_new();
+  rb_gc_register_mark_object(to_resume_requests);
+
+  sym_accept = ID2SYM(rb_intern("accept"));
   sym_term_close = ID2SYM(rb_intern("term_close"));
   sym_writing = ID2SYM(rb_intern("writing"));
   sym_reading = ID2SYM(rb_intern("reading"));
   sym_sleep = ID2SYM(rb_intern("sleep"));
 
-  to_resume_actions = rb_ary_new();
-  rb_gc_register_mark_object(to_resume_actions);
-
   rb_define_singleton_method(ext, "init_queue", ext_init_queue, 0);
   rb_define_singleton_method(ext, "run_queue", ext_run_queue, 1);
   rb_define_singleton_method(ext, "graceful_quit", ext_graceful_quit, 1);
@@ -390,6 +399,7 @@ void Init_event(VALUE ext) {
   // fd operations
   rb_define_singleton_method(ext, "set_nonblock", ext_set_nonblock, 1);
   rb_define_singleton_method(ext, "fd_watch", ext_fd_watch, 1);
+  rb_define_singleton_method(ext, "fd_unwatch", ext_fd_unwatch, 1);
   rb_define_singleton_method(ext, "fd_send", ext_fd_send, 3);
   rb_define_singleton_method(ext, "fd_recv", ext_fd_recv, 3);
 

data/ext/extconf.rb

@@ -19,6 +19,13 @@ def tweak_cflags
   puts "To enable debug: make xflags='-DDEBUG -O0'"
 end
 
+# for debugging low versions of gcc
+def use_gcc42
+  RbConfig::MAKEFILE_CONFIG['CC'] = `which gcc-4.2`.strip
+  RbConfig::MAKEFILE_CONFIG['CXX'] = `which g++-4.2`.strip
+end
+# use_gcc42
+
 have_kqueue = (have_header("sys/event.h") and have_header("sys/queue.h"))
 have_epoll = have_func('epoll_create', 'sys/epoll.h')
 abort('no kqueue nor epoll') if !have_kqueue and !have_epoll

data/ext/inc/epoll.h

@@ -6,35 +6,24 @@
 
 static struct epoll_event qevents[MAX_E];
 
-static void ADD_E(int fd, uint64_t etype) {
+static void ADD_E(int fd, VALUE rid) {
   struct epoll_event e;
   // not using edge trigger flag EPOLLET
   // because edge trigger only fire once when fd is readable/writable
   // but the event may not be consumed in our handler
   e.events = EPOLLIN | EPOLLOUT;
-  e.data.u64 = (etype << 32) | (uint64_t)fd;
+  e.data.u64 = (uint64_t)rid;
 
-  // todo timeout
-# ifdef NDEBUG
-  epoll_ctl(qfd, EPOLL_CTL_ADD, fd, &e);
-# else
   if (epoll_ctl(qfd, EPOLL_CTL_ADD, fd, &e))
-    printf("%s: %s\n", __func__, strerror(errno));
-# endif
+    rb_sys_fail("epoll_ctl(2) - EPOLL_CTL_ADD");
 }
 
 // NOTE either epoll or kqueue removes the event watch from queue when fd closed
 static void DEL_E(int fd) {
   struct epoll_event e;
   e.events = EPOLLIN | EPOLLOUT;
-  e.data.ptr = NULL;
-
-# ifdef NDEBUG
-  epoll_ctl(qfd, EPOLL_CTL_DEL, fd, &e);
-# else
   if (epoll_ctl(qfd, EPOLL_CTL_DEL, fd, &e))
-    printf("%s: %s\n", __func__, strerror(errno));
-# endif
+    rb_sys_fail("epoll_ctl(2) - EPOLL_CTL_DEL");
 }
 
 static void INIT_E() {
@@ -48,13 +37,16 @@ static void LOOP_E() {
   while (1) {
     // heart beat of 0.1 sec, allow ruby signal interrupts to be inserted
     int sz = epoll_wait(qfd, qevents, MAX_E, 100);
+    st_clear(handled_rids);
 
     for (int i = 0; i < sz; i++) {
-      if (qevents[i].events & (EPOLLIN | EPOLLOUT)) {
-        int fd = (int)(qevents[i].data.u64 & 0xFFFFFFFF);
-        int etype = (int)(qevents[i].data.u64 >> 32);
-        loop_body(fd, etype);
-        break;
+      VALUE rid = (VALUE)qevents[i].data.u64;
+      if (qevents[i].events & (EPOLLHUP | EPOLLERR)) {
+        nyara_detach_rid(rid);
+      } else if (qevents[i].events & (EPOLLIN | EPOLLOUT)) {
+        loop_body(rid);
+      } else if (qevents[i].events & EPOLLRDHUP) {
+        // do sth?
       }
     }
     loop_check();

data/ext/inc/kqueue.h

@@ -10,32 +10,22 @@
 # include <sys/queue.h>
 #endif
 
-#define MAX_E 1024
-static int qfd;
 static struct kevent qevents[MAX_E];
 
-static void ADD_E(int fd, uint64_t etype) {
+static void ADD_E(int fd, VALUE rid) {
   struct kevent e;
   // without EV_CLEAR, it is level-triggered
   // http://www.cs.helsinki.fi/linux/linux-kernel/2001-38/0547.html
-  EV_SET(&e, fd, EVFILT_READ | EVFILT_WRITE, EV_ADD, 0, 0, (void*)etype);
-# ifdef NDEBUG
-  kevent(qfd, &e, 1, NULL, 0, NULL);
-# else
+  EV_SET(&e, fd, EVFILT_READ | EVFILT_WRITE, EV_ADD, 0, 0, (void*)rid);
   if (kevent(qfd, &e, 1, NULL, 0, NULL))
-    printf("%s: %s\n", __func__, strerror(errno));
-# endif
+    rb_sys_fail("kevent(2) - EV_ADD");
 }
 
 static void DEL_E_WITH_FILTER(int fd, int filter) {
   struct kevent e;
   EV_SET(&e, fd, filter, EV_DELETE, 0, 0, NULL);
-# ifdef NDEBUG
-  kevent(qfd, &e, 1, NULL, 0, NULL);
-# else
   if (kevent(qfd, &e, 1, NULL, 0, NULL))
-    printf("%s: %s\n", __func__, strerror(errno));
-# endif
+    rb_sys_fail("kevent(2) - EV_DELETE");
 }
 
 static void DEL_E(int fd) {
@@ -55,21 +45,21 @@ static void LOOP_E() {
   // EV_DELETE, EV_RECEIPT, EV_ONESHOT,
   // EV_CLEAR, EV_EOF, EV_ERROR);
 
-  struct timespec ts = {0, 1000 * 1000 * 100};
+  static struct timespec ts = {0, 1000 * 1000 * 100};
   while (1) {
     // heart beat of 0.1 sec, allow ruby signal interrupts to be inserted
     int sz = kevent(qfd, NULL, 0, qevents, MAX_E, &ts);
+    st_clear(handled_rids);
 
     for (int i = 0; i < sz; i++) {
-      int fd = (int)qevents[i].ident;
       if (qevents[i].flags & EV_EOF) {
+        int fd = (int)qevents[i].ident;
         // EV_EOF is set if the read side of the socket is shutdown
         // the event can keep flipping back to consume cpu if we don't remove it
         DEL_E_WITH_FILTER(fd, qevents[i].filter);
       }
       if (qevents[i].filter & (EVFILT_READ | EVFILT_WRITE)) {
-        loop_body(fd, (int)qevents[i].udata);
-        break;
+        loop_body((VALUE)qevents[i].udata);
       }
     }
     loop_check();

data/ext/nyara.c

@@ -7,6 +7,7 @@
 #include <sys/fcntl.h>
 
 rb_encoding* u8_encoding;
+static VALUE nyara;
 
 void nyara_set_nonblock(int fd) {
   int flags;
@@ -30,11 +31,24 @@ static void set_fd_limit(int nofiles) {
   }
 }
 
+static bool summary_request = false;
+void nyara_summary_request(int method, VALUE path, VALUE controller) {
+  if (summary_request) {
+    rb_funcall(nyara, rb_intern("summary_request"), 3, INT2NUM(method), path, controller);
+  }
+}
+
+// set whether should log summary of request
+static VALUE ext_summary_request(VALUE _, VALUE toggle) {
+  summary_request = RTEST(toggle);
+  return toggle;
+}
+
 void Init_nyara() {
   u8_encoding = rb_utf8_encoding();
   set_fd_limit(20000);
 
-  VALUE nyara = rb_define_module("Nyara");
+  nyara = rb_define_module("Nyara");
 # include "inc/version.inc"
   rb_const_set(nyara, rb_intern("VERSION"), rb_enc_str_new(NYARA_VERSION, strlen(NYARA_VERSION), u8_encoding));
 
@@ -66,10 +80,11 @@ void Init_nyara() {
   OBJ_FREEZE(status_map);
 
   VALUE ext = rb_define_module_under(nyara, "Ext");
+  rb_define_singleton_method(ext, "summary_request", ext_summary_request, 1);
   Init_accept(ext);
   Init_mime(ext);
   Init_request(nyara, ext);
-  Init_request_parse(nyara);
+  Init_request_parse(nyara, ext);
   Init_test_response(nyara);
   Init_event(ext);
   Init_route(nyara, ext);

data/ext/nyara.h

@@ -9,19 +9,22 @@
 #undef NDEBUG
 #endif
 
+#define nyara_inspect(value) do {\
+    volatile VALUE _xx = rb_inspect(value);\
+    printf("%s: %.*s\n", __func__, (int)RSTRING_LEN(_xx), RSTRING_PTR(_xx));\
+  } while(0)
 
 /* event.c */
 void Init_event(VALUE ext);
-void nyara_detach_fd(int fd);
+void nyara_detach_rid(VALUE rid);
 
 
 /* request_parse.c */
-void Init_request_parse(VALUE nyara);
+void Init_request_parse(VALUE nyara, VALUE ext);
 
 
 /* request.c */
 void Init_request(VALUE nyara, VALUE ext);
-VALUE nyara_request_new(int fd);
 void nyara_request_init_env(VALUE request);
 void nyara_request_term_close(VALUE request);
 bool nyara_send_data(int fd, const char* s, long len);
@@ -75,4 +78,5 @@ extern RouteResult nyara_lookup_route(enum http_method method_num, VALUE vpath,
 
 /* nyara.c */
 void nyara_set_nonblock(int fd);
+void nyara_summary_request(int method, VALUE path, VALUE controller);
 extern rb_encoding* u8_encoding;