agoo 2.5.1 → 2.5.2

data/ext/agoo/con.h

@@ -4,23 +4,28 @@
 #define __AGOO_CON_H__
 
 #include <poll.h>
+#include <pthread.h>
 #include <stdbool.h>
 #include <stdint.h>
 
 #include "err.h"
-#include "request.h"
+#include "req.h"
 #include "response.h"
 #include "server.h"
-#include "types.h"
+#include "kinds.h"
 
 #define MAX_HEADER_SIZE	8192
 
 struct _Upgraded;
+struct _Req;
+struct _Res;
+struct _Bind;
+struct _Queue;
 
 typedef struct _Con {
     struct _Con		*next;
     int			sock;
-    ConKind		kind;
+    struct _Bind	*bind;
     struct pollfd	*pp;
     uint64_t		id;
     char		buf[MAX_HEADER_SIZE];
@@ -33,17 +38,28 @@ typedef struct _Con {
     bool		closing;
     bool		dead;
     volatile bool	hijacked;
-    Req			req;
-    Res			res_head;
-    Res			res_tail;
+    struct _Req		*req;
+    struct _Res		*res_head;
+    struct _Res		*res_tail;
 
     struct _Upgraded	*up; // only set for push connections
 } *Con;
 
-extern Con		con_create(Err err, int sock, uint64_t id);
+typedef struct _ConLoop {
+    struct _ConLoop	*next;
+    struct _Queue	pub_queue;
+    pthread_t		thread;
+    int			id;
+} *ConLoop;
+    
+extern Con		con_create(Err err, int sock, uint64_t id, struct _Bind *b);
 extern void		con_destroy(Con c);
 extern const char*	con_header_value(const char *header, int hlen, const char *key, int *vlen);
 
-extern void*		con_loop(void *ctx);
+extern struct _ConLoop*	conloop_create(Err err, int id);
+
+extern bool		con_http_read(Con c);
+extern bool		con_http_write(Con c);
+extern short		con_http_events(Con c);
 
 #endif /* __AGOO_CON_H__ */

data/ext/agoo/debug.c

@@ -35,6 +35,7 @@ atomic_int	mem_hook_pattern = 0;
 atomic_int	mem_http_slot = 0;
 atomic_int	mem_log_entry = 0;
 atomic_int	mem_log_what = 0;
+atomic_int	mem_log_tid = 0;
 atomic_int	mem_mime_slot = 0;
 atomic_int	mem_page = 0;
 atomic_int	mem_page_msg = 0;
@@ -102,6 +103,7 @@ print_stats() {
     printf("  mem_http_slot:    %d\n", mem_http_slot);
     printf("  mem_log_entry:    %d\n", mem_log_entry);
     printf("  mem_log_what:     %d\n", mem_log_what);
+    printf("  mem_log_tid:      %d\n", mem_log_tid);
     printf("  mem_mime_slot:    %d\n", mem_mime_slot);
     printf("  mem_page:         %d\n", mem_page);
     printf("  mem_page_msg:     %d\n", mem_page_msg);

data/ext/agoo/debug.h

@@ -29,6 +29,7 @@ extern atomic_int	mem_hook_pattern;
 extern atomic_int	mem_http_slot;
 extern atomic_int	mem_log_entry;
 extern atomic_int	mem_log_what;
+extern atomic_int	mem_log_tid;
 extern atomic_int	mem_mime_slot;
 extern atomic_int	mem_page;
 extern atomic_int	mem_page_msg;

data/ext/agoo/err.c

@@ -54,7 +54,7 @@ const char*
 err_str(ErrCode code) {
     const char	*str = NULL;
     
-    if (code <= ELAST) {
+    if (code < ERR_START) {
 	str = strerror(code);
     }
     if (NULL == str) {

data/ext/agoo/err.h

@@ -5,10 +5,7 @@
 
 #include <errno.h>
 
-#ifndef ELAST
-#define ELAST 300
-#endif
-
+#define ERR_START	300
 #define ERR_INIT	{ 0, { 0 } }
 
 typedef enum {
@@ -16,7 +13,7 @@ typedef enum {
     ERR_MEMORY	= ENOMEM,
     ERR_DENIED	= EACCES,
     ERR_IMPL	= ENOSYS,
-    ERR_PARSE	= ELAST + 1,
+    ERR_PARSE	= ERR_START,
     ERR_READ,
     ERR_WRITE,
     ERR_OVERFLOW,

data/ext/agoo/foo/agoo.c

@@ -0,0 +1,109 @@
+// Copyright (c) 2018, Peter Ohler, All rights reserved.
+
+#include <signal.h>
+#include <stdio.h>
+
+#include <ruby.h>
+
+#include "debug.h"
+#include "error_stream.h"
+#include "pub.h"
+#include "rack_logger.h"
+#include "request.h"
+#include "rresponse.h"
+#include "rlog.h"
+#include "rserver.h"
+#include "rupgraded.h"
+#include "server.h"
+#include "upgraded.h"
+
+void
+agoo_shutdown() {
+    rserver_shutdown(Qnil);
+    log_close();
+}
+
+/* Document-method: shutdown
+ *
+ * call-seq: shutdown()
+ *
+ * Shutdown the server and logger.
+ */
+static VALUE
+ragoo_shutdown(VALUE self) {
+    agoo_shutdown();
+    debug_rreport();
+    return Qnil;
+}
+
+/* Document-method: publish
+ *
+ * call-seq: publish(subject, message)
+ *
+ * Publish a message on the given subject. A subject is normally a String but
+ * Symbols can also be used as can any other object that responds to #to_s.
+ */
+VALUE
+ragoo_publish(VALUE self, VALUE subject, VALUE message) {
+    int		slen;
+    const char	*subj = extract_subject(subject, &slen);
+
+    rb_check_type(message, T_STRING);
+    server_publish(pub_publish(subj, slen, StringValuePtr(message), (int)RSTRING_LEN(message)));
+
+    return Qnil;
+}
+
+/* Document-method: unsubscribe
+ *
+ * call-seq: unsubscribe(subject)
+ *
+ * Unsubscribes on client listeners on the specified subject. Subjects are
+ * normally Strings but Symbols can also be used as can any other object that
+ * responds to #to_s.
+ */
+static VALUE
+ragoo_unsubscribe(VALUE self, VALUE subject) {
+    rb_check_type(subject, T_STRING);
+
+    server_publish(pub_unsubscribe(NULL, StringValuePtr(subject), (int)RSTRING_LEN(subject)));
+
+    return Qnil;
+}
+
+static void
+sig_handler(int sig) {
+    agoo_shutdown();
+
+    debug_report();
+    // Use exit instead of rb_exit as rb_exit segfaults most of the time.
+    //rb_exit(0);
+    exit(0);
+}
+
+
+/* Document-module: Agoo
+ *
+ * A High Performance HTTP Server that supports the Ruby rack API. The word
+ * agoo is a Japanese word for a type of flying fish.
+ */
+void
+Init_agoo() {
+    VALUE	mod = rb_define_module("Agoo");
+
+    rlog_init(mod);
+    error_stream_init(mod);
+    rack_logger_init(mod);
+    request_init(mod);
+    response_init(mod);
+    server_init(mod);
+    upgraded_init(mod);
+
+    rb_define_module_function(mod, "shutdown", ragoo_shutdown, 0);
+    rb_define_module_function(mod, "publish", ragoo_publish, 2);
+    rb_define_module_function(mod, "unsubscribe", ragoo_unsubscribe, 1);
+
+    signal(SIGINT, sig_handler);
+    signal(SIGTERM, sig_handler);
+    signal(SIGPIPE, SIG_IGN);
+}

data/ext/agoo/foo/con.c

@@ -0,0 +1,1220 @@
+// Copyright (c) 2018, Peter Ohler, All rights reserved.
+
+#include <ctype.h>
+#include <netdb.h>
+#include <stdio.h>
+#include <string.h>
+#include <unistd.h>
+
+#include "bind.h"
+#include "con.h"
+#include "debug.h"
+#include "dtime.h"
+#include "hook.h"
+#include "http.h"
+#include "log.h"
+#include "page.h"
+#include "pub.h"
+#include "res.h"
+#include "seg.h"
+#include "server.h"
+#include "sse.h"
+#include "subject.h"
+#include "upgraded.h"
+#include "websocket.h"
+
+#define CON_TIMEOUT		5.0
+#define INITIAL_POLL_SIZE	1024
+
+static bool	con_ws_read(Con c);
+static bool	con_ws_write(Con c);
+static short	con_ws_events(Con c);
+static bool	con_sse_write(Con c);
+static short	con_sse_events(Con c);
+
+static struct _Bind	ws_bind = {
+    .kind = CON_WS,
+    .read = con_ws_read,
+    .write = con_ws_write,
+    .events = con_ws_events,
+};
+
+static struct _Bind	sse_bind = {
+    .kind = CON_SSE,
+    .read = NULL,
+    .write = con_sse_write,
+    .events = con_sse_events,
+};
+
+Con
+con_create(Err err, int sock, uint64_t id, Bind b) {
+    Con	c;
+
+    if (NULL == (c = (Con)malloc(sizeof(struct _Con)))) {
+	err_set(err, ERR_MEMORY, "Failed to allocate memory for a connection.");
+    } else {
+	DEBUG_ALLOC(mem_con, c)
+	memset(c, 0, sizeof(struct _Con));
+	c->sock = sock;
+	c->id = id;
+	c->timeout = dtime() + CON_TIMEOUT;
+	c->bind = b;
+    }
+    return c;
+}
+
+void
+con_destroy(Con c) {
+    atomic_fetch_sub(&the_server.con_cnt, 1);
+
+    if (CON_WS == c->bind->kind || CON_SSE == c->bind->kind) {
+	ws_req_close(c);
+    }
+    if (0 < c->sock) {
+	close(c->sock);
+	c->sock = 0;
+    }
+    if (NULL != c->req) {
+	req_destroy(c->req);
+    }
+    if (NULL != c->up) {
+	upgraded_release_con(c->up);
+	c->up = NULL;
+    }
+    DEBUG_FREE(mem_con, c)
+    free(c);
+}
+
+const char*
+con_header_value(const char *header, int hlen, const char *key, int *vlen) {
+    // Search for \r then check for \n and then the key followed by a :. Keep
+    // trying until the end of the header.
+    const char	*h = header;
+    const char	*hend = header + hlen;
+    const char	*value;
+    int		klen = (int)strlen(key);
+    
+    while (h < hend) {
+	if (0 == strncmp(key, h, klen) && ':' == h[klen]) {
+	    h += klen + 1;
+	    for (; ' ' == *h; h++) {
+	    }
+	    value = h;
+	    for (; '\r' != *h && '\0' != *h; h++) {
+	    }
+	    *vlen = (int)(h - value);
+ 
+	    return value;
+	}
+	for (; h < hend; h++) {
+	    if ('\r' == *h && '\n' == *(h + 1)) {
+		h += 2;
+		break;
+	    }
+	}
+    }
+    return NULL;
+}
+
+static long
+bad_request(Con c, int status, int line) {
+    Res		res;
+    const char *msg = http_code_message(status);
+    
+    if (NULL == (res = res_create(c))) {
+	log_cat(&error_cat, "memory allocation of response failed on connection %llu @ %d.", (unsigned long long)c->id, line);
+    } else {
+	char	buf[256];
+	int	cnt = snprintf(buf, sizeof(buf), "HTTP/1.1 %d %s\r\nConnection: Close\r\nContent-Length: 0\r\n\r\n", status, msg);
+	Text	message = text_create(buf, cnt);
+	
+	if (NULL == c->res_tail) {
+	    c->res_head = res;
+	} else {
+	    c->res_tail->next = res;
+	}
+	c->res_tail = res;
+	res->close = true;
+	res_set_message(res, message);
+    }
+    return -1;
+}
+
+static bool
+should_close(const char *header, int hlen) {
+    const char	*v;
+    int		vlen = 0;
+    
+    if (NULL != (v = con_header_value(header, hlen, "Connection", &vlen))) {
+	return (5 == vlen && 0 == strncasecmp("Close", v, 5));
+    }
+    return false;
+}
+
+static bool
+page_response(Con c, Page p, char *hend) {
+    Res 	res;
+    char	*b;
+    
+    if (NULL == (res = res_create(c))) {
+	return true;
+    }
+    if (NULL == c->res_tail) {
+	c->res_head = res;
+    } else {
+	c->res_tail->next = res;
+    }
+    c->res_tail = res;
+
+    b = strstr(c->buf, "\r\n");
+    res->close = should_close(b, (int)(hend - b));
+    if (res->close) {
+	c->closing = true;
+    }
+    res_set_message(res, p->resp);
+
+    return false;
+}
+
+// rserver
+static void
+push_error(Upgraded up, const char *msg, int mlen) {
+    if (NULL != up && the_server.ctx_nil_value != up->ctx && up->on_error) {
+	Req	req = req_create(mlen);
+
+	if (NULL == req) {
+	    return;
+	}
+	memcpy(req->msg, msg, mlen);
+	req->msg[mlen] = '\0';
+	req->up = up;
+	req->method = ON_ERROR;
+	req->hook = hook_create(NONE, NULL, up->ctx, PUSH_HOOK, &the_server.eval_queue);
+	upgraded_ref(up);
+	queue_push(&the_server.eval_queue, (void*)req);
+    }
+}
+
+// Returns:
+//  0 - when header has not been read
+//  message length - when length can be determined
+//  -1 on a bad request
+//  negative of message length - when message is handled here.
+static long
+con_header_read(Con c) {
+    char	*hend = strstr(c->buf, "\r\n\r\n");
+    Method	method;
+    struct _Seg	path;
+    char	*query = NULL;
+    char	*qend;
+    char	*b;
+    size_t	clen = 0;
+    long	mlen;
+    Hook	hook = NULL;
+    Page	p;
+    struct _Err	err = ERR_INIT;
+    
+    if (NULL == hend) {
+	if (sizeof(c->buf) - 1 <= c->bcnt) {
+	    return bad_request(c, 431, __LINE__);
+	}
+	return 0;
+    }
+    if (req_cat.on) {
+	*hend = '\0';
+	log_cat(&req_cat, "%llu: %s", (unsigned long long)c->id, c->buf);
+	*hend = '\r';
+    }
+    for (b = c->buf; ' ' != *b; b++) {
+	if ('\0' == *b) {
+	    return bad_request(c, 400, __LINE__);
+	}
+    }
+    switch (toupper(*c->buf)) {
+    case 'G':
+	if (3 != b - c->buf || 0 != strncmp("GET", c->buf, 3)) {
+	    return bad_request(c, 400, __LINE__);
+	}
+	method = GET;
+	break;
+    case 'P': {
+	const char	*v;
+	int		vlen = 0;
+	char		*vend;
+	
+	if (3 == b - c->buf && 0 == strncmp("PUT", c->buf, 3)) {
+	    method = PUT;
+	} else if (4 == b - c->buf && 0 == strncmp("POST", c->buf, 4)) {
+	    method = POST;
+	} else {
+	    return bad_request(c, 400, __LINE__);
+	}
+	if (NULL == (v = con_header_value(c->buf, (int)(hend - c->buf), "Content-Length", &vlen))) {
+	    return bad_request(c, 411, __LINE__);
+	}
+	clen = (size_t)strtoul(v, &vend, 10);
+	if (vend != v + vlen) {
+	    return bad_request(c, 411, __LINE__);
+	}
+	break;
+    }
+    case 'D':
+	if (6 != b - c->buf || 0 != strncmp("DELETE", c->buf, 6)) {
+	    return bad_request(c, 400, __LINE__);
+	}
+	method = DELETE;
+	break;
+    case 'H':
+	if (4 != b - c->buf || 0 != strncmp("HEAD", c->buf, 4)) {
+	    return bad_request(c, 400, __LINE__);
+	}
+	method = HEAD;
+	break;
+    case 'O':
+	if (7 != b - c->buf || 0 != strncmp("OPTIONS", c->buf, 7)) {
+	    return bad_request(c, 400, __LINE__);
+	}
+	method = OPTIONS;
+	break;
+    case 'C':
+	if (7 != b - c->buf || 0 != strncmp("CONNECT", c->buf, 7)) {
+	    return bad_request(c, 400, __LINE__);
+	}
+	method = CONNECT;
+	break;
+    default:
+	return bad_request(c, 400, __LINE__);
+    }
+    for (; ' ' == *b; b++) {
+	if ('\0' == *b) {
+	    return bad_request(c, 400, __LINE__);
+	}
+    }
+    path.start = b;
+    for (; ' ' != *b; b++) {
+	switch (*b) {
+	case '?':
+	    path.end = b;
+	    query = b + 1;
+	    break;
+	case '\0':
+	    return bad_request(c, 400, __LINE__);
+	default:
+	    break;
+	}
+    }
+    if (NULL == query) {
+	path.end = b;
+	query = b;
+	qend = b;
+    } else {
+	qend = b;
+    }
+    mlen = hend - c->buf + 4 + clen;
+    if (GET == method &&
+	NULL != (p = group_get(&err, path.start, (int)(path.end - path.start)))) {
+	if (page_response(c, p, hend)) {
+	    return bad_request(c, 500, __LINE__);
+	}
+	return -mlen;
+    }
+    if (GET == method && the_server.root_first &&
+	NULL != (p = page_get(&err, path.start, (int)(path.end - path.start)))) {
+	if (page_response(c, p, hend)) {
+	    return bad_request(c, 500, __LINE__);
+	}
+	return -mlen;
+    }
+    if (NULL == (hook = hook_find(the_server.hooks, method, &path))) {
+	if (GET == method) {
+	    if (the_server.root_first) { // already checked
+		if (NULL != the_server.hook404) {
+		    // There would be too many parameters to pass to a
+		    // separate function so just goto the hook processing.
+		    hook = the_server.hook404;
+		    goto HOOKED;
+		}
+		return bad_request(c, 404, __LINE__);
+	    }
+	    if (NULL == (p = page_get(&err, path.start, (int)(path.end - path.start)))) {
+		if (NULL != the_server.hook404) {
+		    // There would be too many parameters to pass to a
+		    // separate function so just goto the hook processing.
+		    hook = the_server.hook404;
+		    goto HOOKED;
+		}
+		return bad_request(c, 404, __LINE__);
+	    }
+	    if (page_response(c, p, hend)) {
+		return bad_request(c, 500, __LINE__);
+	    }
+	    return -mlen;
+	}
+ 	return bad_request(c, 404, __LINE__);
+   }
+HOOKED:
+    // Create request and populate.
+    if (NULL == (c->req = req_create(mlen))) {
+	return bad_request(c, 413, __LINE__);
+    }
+    if ((long)c->bcnt <= mlen) {
+	memcpy(c->req->msg, c->buf, c->bcnt);
+	if ((long)c->bcnt < mlen) {
+	    memset(c->req->msg + c->bcnt, 0, mlen - c->bcnt);
+	}
+    } else {
+	memcpy(c->req->msg, c->buf, mlen);
+    }
+    c->req->msg[mlen] = '\0';
+    c->req->method = method;
+    c->req->upgrade = UP_NONE;
+    c->req->up = NULL;
+    c->req->path.start = c->req->msg + (path.start - c->buf);
+    c->req->path.len = (int)(path.end - path.start);
+    c->req->query.start = c->req->msg + (query - c->buf);
+    c->req->query.len = (int)(qend - query);
+    c->req->query.start[c->req->query.len] = '\0';
+    c->req->body.start = c->req->msg + (hend - c->buf + 4);
+    c->req->body.len = (unsigned int)clen;
+    b = strstr(b, "\r\n");
+    c->req->header.start = c->req->msg + (b + 2 - c->buf);
+    c->req->header.len = (unsigned int)(hend - b - 2);
+    c->req->res = NULL;
+    c->req->hook = hook;
+
+    return mlen;
+}
+
+static void
+check_upgrade(Con c) {
+    const char	*v;
+    int		vlen = 0;
+
+    if (NULL == c->req) {
+	return;
+    }
+    if (NULL != (v = con_header_value(c->req->header.start, c->req->header.len, "Connection", &vlen))) {
+	if (NULL != strstr(v, "Upgrade")) {
+	    if (NULL != (v = con_header_value(c->req->header.start, c->req->header.len, "Upgrade", &vlen))) {
+		if (0 == strncasecmp("WebSocket", v, vlen)) {
+		    c->res_tail->close = false;
+		    c->res_tail->con_kind = CON_WS;
+		    return;
+		}
+	    }
+	}
+    }
+    if (NULL != (v = con_header_value(c->req->header.start, c->req->header.len, "Accept", &vlen))) {
+	if (0 == strncasecmp("text/event-stream", v, vlen)) {
+	    c->res_tail->close = false;
+	    c->res_tail->con_kind = CON_SSE;
+	    return;
+	}
+    }
+}
+
+bool
+con_http_read(Con c) {
+    ssize_t	cnt;
+
+    if (c->dead || 0 == c->sock || c->closing) {
+	return true;
+    }
+    if (NULL != c->req) {
+	cnt = recv(c->sock, c->req->msg + c->bcnt, c->req->mlen - c->bcnt, 0);
+    } else {
+	cnt = recv(c->sock, c->buf + c->bcnt, sizeof(c->buf) - c->bcnt - 1, 0);
+    }
+    c->timeout = dtime() + CON_TIMEOUT;
+    if (0 >= cnt) {
+	// If nothing read then no need to complain. Just close.
+	if (0 < c->bcnt) {
+	    if (0 == cnt) {
+		log_cat(&warn_cat, "Nothing to read. Client closed socket on connection %llu.", (unsigned long long)c->id);
+	    } else {
+		log_cat(&warn_cat, "Failed to read request. %s.", strerror(errno));
+	    }
+	}
+	return true;
+    }
+    c->bcnt += cnt;
+    while (true) {
+	if (NULL == c->req) {
+	    long	mlen;
+	    
+	    // Terminate with \0 for debug and strstr() check
+	    c->buf[c->bcnt] = '\0';
+	    if (0 > (mlen = con_header_read(c))) {
+		if (-1 == mlen) {
+		    c->bcnt = 0;
+		    *c->buf = '\0';
+		    return false;
+		} else if (-mlen < (long)c->bcnt) {
+		    mlen = -mlen;
+		    memmove(c->buf, c->buf + mlen, c->bcnt - mlen);
+		    c->bcnt -= mlen;
+		} else {
+		    c->bcnt = 0;
+		    *c->buf = '\0';
+		    return false;
+		}
+		continue;
+	    }
+	}
+	if (NULL != c->req) {
+	    if (c->req->mlen <= c->bcnt) {
+		Req	req;
+		Res	res;
+		long	mlen;
+
+		if (debug_cat.on && NULL != c->req && NULL != c->req->body.start) {
+		    log_cat(&debug_cat, "request on %llu: %s", (unsigned long long)c->id, c->req->body.start);
+		}
+		if (NULL == (res = res_create(c))) {
+		    c->req = NULL;
+		    log_cat(&error_cat, "memory allocation of response failed on connection %llu.", (unsigned long long)c->id);
+		    return bad_request(c, 500, __LINE__);
+		} else {
+		    if (NULL == c->res_tail) {
+			c->res_head = res;
+		    } else {
+			c->res_tail->next = res;
+		    }
+		    c->res_tail = res;
+		    res->close = should_close(c->req->header.start, c->req->header.len);
+		    if (res->close) {
+			c->closing = true;
+		    }
+		}
+		c->req->res = res;
+		mlen = c->req->mlen;
+		check_upgrade(c);
+		req = c->req;
+		c->req = NULL;
+		queue_push(req->hook->queue, (void*)req);
+		if (mlen < (long)c->bcnt) {
+		    memmove(c->buf, c->buf + mlen, c->bcnt - mlen);
+		    c->bcnt -= mlen;
+		} else {
+		    c->bcnt = 0;
+		    *c->buf = '\0';
+		    break;
+		}
+		continue;
+	    }
+	}
+	break;
+    }
+    return false;
+}
+
+static bool
+con_ws_read(Con c) {
+    ssize_t	cnt;
+    uint8_t	*b;
+    uint8_t	op;
+    long	mlen;	
+
+    if (NULL != c->req) {
+	cnt = recv(c->sock, c->req->msg + c->bcnt, c->req->mlen - c->bcnt, 0);
+    } else {
+	cnt = recv(c->sock, c->buf + c->bcnt, sizeof(c->buf) - c->bcnt - 1, 0);
+    }
+    c->timeout = dtime() + CON_TIMEOUT;
+    if (0 >= cnt) {
+	// If nothing read then no need to complain. Just close.
+	if (0 < c->bcnt) {
+	    if (0 == cnt) {
+		log_cat(&warn_cat, "Nothing to read. Client closed socket on connection %llu.", (unsigned long long)c->id);
+	    } else {
+		char	msg[1024];
+		int	len = snprintf(msg, sizeof(msg) - 1, "Failed to read WebSocket message. %s.", strerror(errno));
+
+		push_error(c->up, msg, len);
+		log_cat(&warn_cat, "Failed to read WebSocket message. %s.", strerror(errno));
+	    }
+	}
+	return true;
+    }
+    c->bcnt += cnt;
+    while (true) {
+	if (NULL == c->req) {
+	    if (c->bcnt < 2) {
+		return false; // Try again.
+	    }
+	    b = (uint8_t*)c->buf;
+	    if (0 >= (mlen = ws_calc_len(c, b, c->bcnt))) {
+		return (mlen < 0);
+	    }
+	    op = 0x0F & *b;
+	    switch (op) {
+	    case WS_OP_TEXT:
+	    case WS_OP_BIN:
+		if (ws_create_req(c, mlen)) {
+		    return true;
+		}
+		break;
+	    case WS_OP_CLOSE:
+		return true;
+	    case WS_OP_PING:
+		if (mlen == (long)c->bcnt) {
+		    ws_pong(c);
+		    c->bcnt = 0;
+		}
+		break;
+	    case WS_OP_PONG:
+		// ignore
+		if (mlen == (long)c->bcnt) {
+		    c->bcnt = 0;
+		}
+		break;
+	    case WS_OP_CONT:
+	    default: {
+		char	msg[1024];
+		int	len = snprintf(msg, sizeof(msg) - 1, "WebSocket op 0x%02x not supported on %llu.",
+				       op, (unsigned long long)c->id);
+
+		push_error(c->up, msg, len);
+		log_cat(&error_cat, "WebSocket op 0x%02x not supported on %llu.", op, (unsigned long long)c->id);
+		return true;
+	    }
+	    }
+	}
+	if (NULL != c->req) {
+	    mlen = c->req->mlen;
+	    c->req->mlen = ws_decode(c->req->msg, c->req->mlen);
+	    if (mlen <= (long)c->bcnt) {
+		if (debug_cat.on) {
+		    if (ON_MSG == c->req->method) {
+			log_cat(&debug_cat, "WebSocket message on %llu: %s", (unsigned long long)c->id, c->req->msg);
+		    } else {
+			log_cat(&debug_cat, "WebSocket binary message on %llu", (unsigned long long)c->id);
+		    }
+		}
+	    }
+	    upgraded_ref(c->up);
+	    queue_push(&the_server.eval_queue, (void*)c->req);
+	    if (mlen < (long)c->bcnt) {
+		memmove(c->buf, c->buf + mlen, c->bcnt - mlen);
+		c->bcnt -= mlen;
+	    } else {
+		c->bcnt = 0;
+		*c->buf = '\0';
+		c->req = NULL;
+		break;
+	    }
+	    c->req = NULL;
+	    continue;
+	}
+	break;
+    }
+    return false;
+}
+
+// return true to remove/close connection
+static bool
+con_read(Con c) {
+    if (NULL != c->bind->read) {
+	return c->bind->read(c);
+    }
+    return true;
+}
+
+// return true to remove/close connection
+bool
+con_http_write(Con c) {
+    Text	message = res_message(c->res_head);
+    ssize_t	cnt;
+
+    c->timeout = dtime() + CON_TIMEOUT;
+    if (0 == c->wcnt) {
+	if (resp_cat.on) {
+	    char	buf[4096];
+	    char	*hend = strstr(message->text, "\r\n\r\n");
+
+	    if (NULL == hend) {
+		hend = message->text + message->len;
+	    }
+	    if ((long)sizeof(buf) <= hend - message->text) {
+		hend = message->text + sizeof(buf) - 1;
+	    }
+	    memcpy(buf, message->text, hend - message->text);
+	    buf[hend - message->text] = '\0';
+	    log_cat(&resp_cat, "%llu: %s", (unsigned long long)c->id, buf);
+	}
+	if (debug_cat.on) {
+	    log_cat(&debug_cat, "response on %llu: %s", (unsigned long long)c->id, message->text);
+	}
+    }
+    if (0 > (cnt = send(c->sock, message->text + c->wcnt, message->len - c->wcnt, MSG_DONTWAIT))) {
+	if (EAGAIN == errno) {
+	    return false;
+	}
+	log_cat(&error_cat, "Socket error @ %llu.", (unsigned long long)c->id);
+
+	return true;
+    }
+    c->wcnt += cnt;
+    if (c->wcnt == message->len) { // finished
+	Res	res = c->res_head;
+	bool	done = res->close;
+	
+	c->res_head = res->next;
+	if (res == c->res_tail) {
+	    c->res_tail = NULL;
+	}
+	c->wcnt = 0;
+	res_destroy(res);
+
+	return done;
+    }
+
+    return false;
+}
+
+static const char	ping_msg[] = "\x89\x00";
+static const char	pong_msg[] = "\x8a\x00";
+
+static bool
+con_ws_write(Con c) {
+    Res		res = c->res_head;
+    Text	message = res_message(res);
+    ssize_t	cnt;
+
+    if (NULL == message) {
+	if (res->ping) {
+	    if (0 > (cnt = send(c->sock, ping_msg, sizeof(ping_msg) - 1, 0))) {
+		char	msg[1024];
+		int	len;
+
+		if (EAGAIN == errno) {
+		    return false;
+		}
+		len = snprintf(msg, sizeof(msg) - 1, "Socket error @ %llu.", (unsigned long long)c->id);
+		push_error(c->up, msg, len);
+		
+		log_cat(&error_cat, "Socket error @ %llu.", (unsigned long long)c->id);
+		ws_req_close(c);
+		res_destroy(res);
+	
+		return true;
+	    }
+	} else if (res->pong) {
+	    if (0 > (cnt = send(c->sock, pong_msg, sizeof(pong_msg) - 1, 0))) {
+		char	msg[1024];
+		int	len;
+
+		if (EAGAIN == errno) {
+		    return false;
+		}
+		len = snprintf(msg, sizeof(msg) - 1, "Socket error @ %llu.", (unsigned long long)c->id);
+		push_error(c->up, msg, len);
+		log_cat(&error_cat, "Socket error @ %llu.", (unsigned long long)c->id);
+		ws_req_close(c);
+		res_destroy(res);
+	
+		return true;
+	    }
+	} else {
+	    ws_req_close(c);
+	    c->res_head = res->next;
+	    if (res == c->res_tail) {
+		c->res_tail = NULL;
+	    }
+	    res_destroy(res);
+	    return true;
+	}
+	c->res_head = res->next;
+	if (res == c->res_tail) {
+	    c->res_tail = NULL;
+	}
+	return false;
+    }
+    c->timeout = dtime() + CON_TIMEOUT;
+    if (0 == c->wcnt) {
+	Text	t;
+	
+	if (push_cat.on) {
+	    if (message->bin) {
+		log_cat(&push_cat, "%llu binary", (unsigned long long)c->id);
+	    } else {
+		log_cat(&push_cat, "%llu: %s", (unsigned long long)c->id, message->text);
+	    }
+	}
+	t = ws_expand(message);
+	if (t != message) {
+	    res_set_message(res, t);
+	    message = t;
+	}
+    }
+    if (0 > (cnt = send(c->sock, message->text + c->wcnt, message->len - c->wcnt, 0))) {
+	char	msg[1024];
+	int	len;
+
+	if (EAGAIN == errno) {
+	    return false;
+	}
+	len = snprintf(msg, sizeof(msg) - 1, "Socket error @ %llu.", (unsigned long long)c->id);
+	push_error(c->up, msg, len);
+	log_cat(&error_cat, "Socket error @ %llu.", (unsigned long long)c->id);
+	ws_req_close(c);
+	
+	return true;
+    }
+    c->wcnt += cnt;
+    if (c->wcnt == message->len) { // finished
+	Res	res = c->res_head;
+	bool	done = res->close;
+	
+	c->res_head = res->next;
+	if (res == c->res_tail) {
+	    c->res_tail = NULL;
+	}
+	c->wcnt = 0;
+	res_destroy(res);
+
+	return done;
+    }
+    return false;
+}
+
+static bool
+con_sse_write(Con c) {
+    Res		res = c->res_head;
+    Text	message = res_message(res);
+    ssize_t	cnt;
+
+    if (NULL == message) {
+	ws_req_close(c);
+	res_destroy(res);
+	return true;
+    }
+    c->timeout = dtime() + CON_TIMEOUT *2;
+    if (0 == c->wcnt) {
+	Text	t;
+	
+	if (push_cat.on) {
+	    log_cat(&push_cat, "%llu: %s", (unsigned long long)c->id, message->text);
+	}
+	t = sse_expand(message);
+	if (t != message) {
+	    res_set_message(res, t);
+	    message = t;
+	}
+    }
+    if (0 > (cnt = send(c->sock, message->text + c->wcnt, message->len - c->wcnt, 0))) {
+	char	msg[1024];
+	int	len;
+
+	if (EAGAIN == errno) {
+	    return false;
+	}
+	len = snprintf(msg, sizeof(msg) - 1, "Socket error @ %llu.", (unsigned long long)c->id);
+	push_error(c->up, msg, len);
+	log_cat(&error_cat, "Socket error @ %llu.", (unsigned long long)c->id);
+	ws_req_close(c);
+	
+	return true;
+    }
+    c->wcnt += cnt;
+    if (c->wcnt == message->len) { // finished
+	Res	res = c->res_head;
+	bool	done = res->close;
+	
+	c->res_head = res->next;
+	if (res == c->res_tail) {
+	    c->res_tail = NULL;
+	}
+	c->wcnt = 0;
+	res_destroy(res);
+
+	return done;
+    }
+    return false;
+}
+
+static bool
+con_write(Con c) {
+    bool	remove = true;
+    ConKind	kind = c->res_head->con_kind;
+
+    if (NULL != c->bind->write) {
+	remove = c->bind->write(c);
+    }
+    //if (kind != c->kind && CON_ANY != kind) {
+    if (CON_ANY != kind) {
+	switch (kind) {
+	case CON_WS:
+	    c->bind = &ws_bind;
+	    break;
+	case CON_SSE:
+	    c->bind = &sse_bind;
+	    break;
+	default:
+	    break;
+	}
+    }
+    return remove;
+}
+
+static void
+publish_pub(Pub pub) {
+    Upgraded	up;
+    const char	*sub = pub->subject->pattern;
+    int	cnt = 0;
+    
+    for (up = the_server.up_list; NULL != up; up = up->next) {
+	if (NULL != up->con && upgraded_match(up, sub)) {
+	    Res	res = res_create(up->con);
+
+	    if (NULL != res) {
+		if (NULL == up->con->res_tail) {
+		    up->con->res_head = res;
+		} else {
+		    up->con->res_tail->next = res;
+		}
+		up->con->res_tail = res;
+		res->con_kind = CON_ANY;
+		res_set_message(res, text_dup(pub->msg));
+		cnt++;
+	    }
+	}
+    }
+}
+
+static void
+unsubscribe_pub(Pub pub) {
+    if (NULL == pub->up) {
+	Upgraded	up;
+
+	for (up = the_server.up_list; NULL != up; up = up->next) {
+	    upgraded_del_subject(up, pub->subject);
+	}
+    } else {
+	upgraded_del_subject(pub->up, pub->subject);
+    }
+}
+
+static void
+process_pub_con(Pub pub) {
+    Upgraded	up = pub->up;
+
+    if (NULL != up) {
+	int	pending;
+	
+	// TBD Change pending to be based on length of con queue
+	if (1 == (pending = atomic_fetch_sub(&up->pending, 1))) {
+	    if (NULL != up && the_server.ctx_nil_value != up->ctx && up->on_empty) {
+		Req	req = req_create(0);
+	    
+		req->up = up;
+		req->method = ON_EMPTY;
+		req->hook = hook_create(NONE, NULL, up->ctx, PUSH_HOOK, &the_server.eval_queue);
+		upgraded_ref(up);
+		queue_push(&the_server.eval_queue, (void*)req);
+	    }
+	}
+    }
+    switch (pub->kind) {
+    case PUB_CLOSE:
+	// A close after already closed is used to decrement the reference
+	// count on the upgraded so it can be destroyed in the con loop
+	// threads.
+	if (NULL != up->con) {
+	    Res	res = res_create(up->con);
+
+	    if (NULL != res) {
+		if (NULL == up->con->res_tail) {
+		    up->con->res_head = res;
+		} else {
+		    up->con->res_tail->next = res;
+		}
+		up->con->res_tail = res;
+		res->con_kind = up->con->bind->kind;
+		res->close = true;
+	    }
+	}
+	break;
+    case PUB_WRITE: {
+	if (NULL == up->con) {
+	    log_cat(&warn_cat, "Connection already closed. WebSocket write failed.");
+	} else {
+	    Res	res = res_create(up->con);
+
+	    if (NULL != res) {
+		if (NULL == up->con->res_tail) {
+		    up->con->res_head = res;
+		} else {
+		    up->con->res_tail->next = res;
+		}
+		up->con->res_tail = res;
+		res->con_kind = CON_ANY;
+		res_set_message(res, pub->msg);
+	    }
+	}
+	break;
+    case PUB_SUB:
+	upgraded_add_subject(pub->up, pub->subject);
+	pub->subject = NULL;
+	break;
+    case PUB_UN:
+	unsubscribe_pub(pub);
+	break;
+    case PUB_MSG:
+	publish_pub(pub);
+	break;
+    }
+    default:
+	break;
+    }
+    pub_destroy(pub);	
+}
+
+short
+con_http_events(Con c) {
+    short	events = 0;
+    
+    if (!c->closing) {
+	events = POLLIN | POLLOUT;
+    }
+    return events;
+}
+
+static short
+con_ws_events(Con c) {
+    short	events = 0;
+
+    if (NULL != c->res_head && (c->res_head->close || c->res_head->ping || NULL != res_message(c->res_head))) {
+	events = POLLIN | POLLOUT;
+    } else if (!c->closing) {
+	events = POLLIN;
+    }
+    return events;
+}
+
+static short
+con_sse_events(Con c) {
+    short	events = 0;
+
+    if (NULL != c->res_head && NULL != res_message(c->res_head)) {
+	events = POLLOUT;
+    }
+    return events;
+}
+
+static struct pollfd*
+poll_setup(Con c, Queue q, struct pollfd *pp) {
+    // The first two pollfd are for the con_queue and the pub_queue in that
+    // order.
+    pp->fd = queue_listen(&the_server.con_queue);
+    pp->events = POLLIN;
+    pp->revents = 0;
+    pp++;
+    pp->fd = queue_listen(q);
+    pp->events = POLLIN;
+    pp->revents = 0;
+    pp++;
+    for (; NULL != c; c = c->next) {
+	if (c->dead || 0 == c->sock) {
+	    continue;
+	}
+	if (c->hijacked) {
+	    c->sock = 0;
+	    continue;
+	}
+	c->pp = pp;
+	pp->fd = c->sock;
+	pp->events = c->bind->events(c);
+	pp->revents = 0;
+	pp++;
+    }
+    return pp;
+}
+
+static bool
+remove_dead_res(Con c) {
+    Res	res;
+
+    while (NULL != (res = c->res_head)) {
+	if (NULL == res_message(c->res_head) && !c->res_head->close && !c->res_head->ping) {
+	    break;
+	}
+	c->res_head = res->next;
+	if (res == c->res_tail) {
+	    c->res_tail = NULL;
+	}
+	res_destroy(res);
+    }
+    return NULL == c->res_head;
+}
+
+void*
+con_loop(void *x) {
+    ConLoop		loop = (ConLoop)x;
+    Con			c;
+    Con			prev;
+    Con			next;
+    Con			cons = NULL;
+    size_t		size = sizeof(struct pollfd) * INITIAL_POLL_SIZE;
+    struct pollfd	*pa = (struct pollfd*)malloc(size);
+    struct pollfd	*pend = pa + INITIAL_POLL_SIZE;
+    struct pollfd	*pp;
+    int			ccnt = 0;
+    int			i;
+    double		now;
+    Pub			pub;
+
+    atomic_fetch_add(&the_server.running, 1);
+    memset(pa, 0, size);
+    while (the_server.active) {
+	while (NULL != (c = (Con)queue_pop(&the_server.con_queue, 0.0))) {
+	    c->next = cons;
+	    cons = c;
+	    ccnt++;
+	    if (pend - pa < ccnt + 2) {
+		size_t	cnt = (pend - pa) * 2;
+
+		size = sizeof(struct pollfd) * cnt;
+		if (NULL == (pa = (struct pollfd*)malloc(size))) {
+		    log_cat(&error_cat, "Out of memory.");
+		    log_close();
+		    exit(EXIT_FAILURE);
+
+		    return NULL;
+		}
+		pend = pa + cnt;
+	    }
+	}
+	while (NULL != (pub = (Pub)queue_pop(&loop->pub_queue, 0.0))) {
+	    process_pub_con(pub);
+	}
+	pp = poll_setup(cons, &loop->pub_queue, pa);
+	if (0 > (i = poll(pa, (nfds_t)(pp - pa), 10))) {
+	    if (EAGAIN == errno) {
+		continue;
+	    }
+	    log_cat(&error_cat, "Polling error. %s.", strerror(errno));
+	    // Either a signal or something bad like out of memory. Might as well exit.
+	    break;
+	}
+	now = dtime();
+	if (0 < i) {
+	    // Check con_queue if an event is waiting.
+	    if (0 != (pa->revents & POLLIN)) {
+		queue_release(&the_server.con_queue);
+		while (NULL != (c = (Con)queue_pop(&the_server.con_queue, 0.0))) {
+		    c->next = cons;
+		    cons = c;
+		    ccnt++;
+		    if (pend - pa < ccnt + 2) {
+			size_t	cnt = (pend - pa) * 2;
+
+			size = sizeof(struct pollfd) * cnt;
+			if (NULL == (pa = (struct pollfd*)malloc(size))) {
+			    log_cat(&error_cat, "Out of memory.");
+			    log_close();
+			    exit(EXIT_FAILURE);
+
+			    return NULL;
+			}
+			pend = pa + cnt;
+		    }
+		}
+	    }
+	    // Check pub_queue if an event is waiting.
+	    if (0 != (pa[1].revents & POLLIN)) {
+		queue_release(&loop->pub_queue);
+		while (NULL != (pub = (Pub)queue_pop(&loop->pub_queue, 0.0))) {
+		    process_pub_con(pub);
+		}
+	    }
+	}
+	prev = NULL;
+	for (c = cons; NULL != c; c = next) {
+	    next = c->next;
+	    if (0 == c->sock || NULL == c->pp) {
+		continue;
+	    }
+	    pp = c->pp;
+	    if (0 != (pp->revents & POLLIN)) {
+		if (con_read(c)) {
+		    c->dead = true;
+		    goto CON_CHECK;
+		}
+	    }
+	    if (0 != (pp->revents & POLLOUT) && NULL != c->res_head && NULL != res_message(c->res_head)) {
+		if (con_write(c)) {
+		    c->dead = true;
+		    goto CON_CHECK;
+		}
+	    }
+	    if (0 != (pp->revents & (POLLERR | POLLHUP | POLLNVAL))) {
+		if (0 < c->bcnt) {
+		    if (0 != (pp->revents & (POLLHUP | POLLNVAL))) {
+			log_cat(&error_cat, "Socket %llu closed.", (unsigned long long)c->id);
+		    } else if (!c->closing) {
+			log_cat(&error_cat, "Socket %llu error. %s", (unsigned long long)c->id, strerror(errno));
+		    }
+		}
+		c->dead = true;
+		goto CON_CHECK;
+	    }
+	CON_CHECK:
+	    if (c->dead || 0 == c->sock) {
+		if (remove_dead_res(c)) {
+		    goto CON_RM;
+		}
+	    } else if (0.0 == c->timeout || now < c->timeout) {
+		prev = c;
+		continue;
+	    } else if (c->closing) {
+		if (remove_dead_res(c)) {
+		    goto CON_RM;
+		}
+	    } else if (CON_WS == c->bind->kind || CON_SSE == c->bind->kind) {
+		c->timeout = dtime() + CON_TIMEOUT;
+		ws_ping(c);
+		continue;
+	    } else {
+		c->closing = true;
+		c->timeout = now + 0.5;
+		prev = c;
+		continue;
+	    }
+	    prev = c;
+	    continue;
+	CON_RM:
+	    if (NULL == prev) {
+		cons = next;
+	    } else {
+		prev->next = next;
+	    }
+	    ccnt--;
+	    log_cat(&con_cat, "Connection %llu closed.", (unsigned long long)c->id);
+	    con_destroy(c);
+	}
+    }
+    while (NULL != (c = cons)) {
+	cons = c->next;
+	con_destroy(c);
+    }
+    atomic_fetch_sub(&the_server.running, 1);
+    
+    return NULL;
+}
+
+ConLoop
+conloop_create(Err err, int id) {
+     ConLoop	loop;
+
+    if (NULL == (loop = (ConLoop)malloc(sizeof(struct _ConLoop)))) {
+	err_set(err, ERR_MEMORY, "Failed to allocate memory for a connection thread.");
+    } else {
+	//DEBUG_ALLOC(mem_con, c);
+	loop->next = NULL;
+	queue_multi_init(&loop->pub_queue, 256, true, false);
+	loop->id = id;
+	pthread_create(&loop->thread, NULL, con_loop, loop);
+    }
+    return loop;
+}