nginxtra 1.6.3.9 → 1.8.0.9

data/vendor/nginx/src/os/unix/ngx_linux_aio_read.c

@@ -24,6 +24,28 @@ io_submit(aio_context_t ctx, long n, struct iocb **paiocb)
 }
 
 
+ngx_int_t
+ngx_file_aio_init(ngx_file_t *file, ngx_pool_t *pool)
+{
+    ngx_event_aio_t  *aio;
+
+    aio = ngx_pcalloc(pool, sizeof(ngx_event_aio_t));
+    if (aio == NULL) {
+        return NGX_ERROR;
+    }
+
+    aio->file = file;
+    aio->fd = file->fd;
+    aio->event.data = aio;
+    aio->event.ready = 1;
+    aio->event.log = file->log;
+
+    file->aio = aio;
+
+    return NGX_OK;
+}
+
+
 ssize_t
 ngx_file_aio_read(ngx_file_t *file, u_char *buf, size_t size, off_t offset,
     ngx_pool_t *pool)
@@ -37,22 +59,11 @@ ngx_file_aio_read(ngx_file_t *file, u_char *buf, size_t size, off_t offset,
         return ngx_read_file(file, buf, size, offset);
     }
 
-    aio = file->aio;
-
-    if (aio == NULL) {
-        aio = ngx_pcalloc(pool, sizeof(ngx_event_aio_t));
-        if (aio == NULL) {
-            return NGX_ERROR;
-        }
-
-        aio->file = file;
-        aio->fd = file->fd;
-        aio->event.data = aio;
-        aio->event.ready = 1;
-        aio->event.log = file->log;
-        file->aio = aio;
+    if (file->aio == NULL && ngx_file_aio_init(file, pool) != NGX_OK) {
+        return NGX_ERROR;
     }
 
+    aio = file->aio;
     ev = &aio->event;
 
     if (!ev->ready) {

data/vendor/nginx/src/os/unix/ngx_linux_config.h

@@ -93,8 +93,11 @@ extern ssize_t sendfile(int s, int fd, int32_t *offset, size_t size);
 #endif
 
 
-#if (NGX_HAVE_FILE_AIO)
+#if (NGX_HAVE_SYS_EVENTFD_H)
+#include <sys/eventfd.h>
+#endif
 #include <sys/syscall.h>
+#if (NGX_HAVE_FILE_AIO)
 #include <linux/aio_abi.h>
 typedef struct iocb  ngx_aiocb_t;
 #endif

data/vendor/nginx/src/os/unix/ngx_linux_sendfile_chain.c

@@ -10,6 +10,22 @@
 #include <ngx_event.h>
 
 
+static ssize_t ngx_linux_sendfile(ngx_connection_t *c, ngx_buf_t *file,
+    size_t size);
+
+#if (NGX_THREADS)
+#include <ngx_thread_pool.h>
+
+#if !(NGX_HAVE_SENDFILE64)
+#error sendfile64() is required!
+#endif
+
+static ngx_int_t ngx_linux_sendfile_thread(ngx_connection_t *c, ngx_buf_t *file,
+    size_t size, size_t *sent);
+static void ngx_linux_sendfile_thread_handler(void *data, ngx_log_t *log);
+#endif
+
+
 /*
  * On Linux up to 2.4.21 sendfile() (syscall #187) works with 32-bit
  * offsets only, and the including <sys/sendfile.h> breaks the compiling,
@@ -27,31 +43,22 @@
 #define NGX_SENDFILE_MAXSIZE  2147483647L
 
 
-#if (IOV_MAX > 64)
-#define NGX_HEADERS  64
-#else
-#define NGX_HEADERS  IOV_MAX
-#endif
-
-
 ngx_chain_t *
 ngx_linux_sendfile_chain(ngx_connection_t *c, ngx_chain_t *in, off_t limit)
 {
-    int            rc, tcp_nodelay;
-    off_t          size, send, prev_send, aligned, sent, fprev;
-    u_char        *prev;
-    size_t         file_size;
+    int            tcp_nodelay;
+    off_t          send, prev_send;
+    size_t         file_size, sent;
+    ssize_t        n;
     ngx_err_t      err;
     ngx_buf_t     *file;
-    ngx_uint_t     eintr, complete;
-    ngx_array_t    header;
     ngx_event_t   *wev;
     ngx_chain_t   *cl;
-    struct iovec  *iov, headers[NGX_HEADERS];
-#if (NGX_HAVE_SENDFILE64)
-    off_t          offset;
-#else
-    int32_t        offset;
+    ngx_iovec_t    header;
+    struct iovec   headers[NGX_IOVS_PREALLOCATE];
+#if (NGX_THREADS)
+    ngx_int_t      rc;
+    ngx_uint_t     thread_handled, thread_complete;
 #endif
 
     wev = c->write;
@@ -70,87 +77,30 @@ ngx_linux_sendfile_chain(ngx_connection_t *c, ngx_chain_t *in, off_t limit)
 
     send = 0;
 
-    header.elts = headers;
-    header.size = sizeof(struct iovec);
-    header.nalloc = NGX_HEADERS;
-    header.pool = c->pool;
+    header.iovs = headers;
+    header.nalloc = NGX_IOVS_PREALLOCATE;
 
     for ( ;; ) {
-        file = NULL;
-        file_size = 0;
-        eintr = 0;
-        complete = 0;
         prev_send = send;
-
-        header.nelts = 0;
-
-        prev = NULL;
-        iov = NULL;
-
-        /* create the iovec and coalesce the neighbouring bufs */
-
-        for (cl = in; cl && send < limit; cl = cl->next) {
-
-            if (ngx_buf_special(cl->buf)) {
-                continue;
-            }
-
-#if 1
-            if (!ngx_buf_in_memory(cl->buf) && !cl->buf->in_file) {
-                ngx_log_error(NGX_LOG_ALERT, c->log, 0,
-                              "zero size buf in sendfile "
-                              "t:%d r:%d f:%d %p %p-%p %p %O-%O",
-                              cl->buf->temporary,
-                              cl->buf->recycled,
-                              cl->buf->in_file,
-                              cl->buf->start,
-                              cl->buf->pos,
-                              cl->buf->last,
-                              cl->buf->file,
-                              cl->buf->file_pos,
-                              cl->buf->file_last);
-
-                ngx_debug_point();
-
-                return NGX_CHAIN_ERROR;
-            }
+#if (NGX_THREADS)
+        thread_handled = 0;
+        thread_complete = 0;
 #endif
 
-            if (!ngx_buf_in_memory_only(cl->buf)) {
-                break;
-            }
-
-            size = cl->buf->last - cl->buf->pos;
-
-            if (send + size > limit) {
-                size = limit - send;
-            }
-
-            if (prev == cl->buf->pos) {
-                iov->iov_len += (size_t) size;
-
-            } else {
-                if (header.nelts >= IOV_MAX) {
-                    break;
-                }
-
-                iov = ngx_array_push(&header);
-                if (iov == NULL) {
-                    return NGX_CHAIN_ERROR;
-                }
+        /* create the iovec and coalesce the neighbouring bufs */
 
-                iov->iov_base = (void *) cl->buf->pos;
-                iov->iov_len = (size_t) size;
-            }
+        cl = ngx_output_chain_to_iovec(&header, in, limit - send, c->log);
 
-            prev = cl->buf->pos + (size_t) size;
-            send += size;
+        if (cl == NGX_CHAIN_ERROR) {
+            return NGX_CHAIN_ERROR;
         }
 
+        send += header.size;
+
         /* set TCP_CORK if there is a header before a file */
 
         if (c->tcp_nopush == NGX_TCP_NOPUSH_UNSET
-            && header.nelts != 0
+            && header.count != 0
             && cl
             && cl->buf->in_file)
         {
@@ -214,165 +164,256 @@ ngx_linux_sendfile_chain(ngx_connection_t *c, ngx_chain_t *in, off_t limit)
 
         /* get the file buf */
 
-        if (header.nelts == 0 && cl && cl->buf->in_file && send < limit) {
+        if (header.count == 0 && cl && cl->buf->in_file && send < limit) {
             file = cl->buf;
 
             /* coalesce the neighbouring file bufs */
 
-            do {
-                size = cl->buf->file_last - cl->buf->file_pos;
-
-                if (send + size > limit) {
-                    size = limit - send;
-
-                    aligned = (cl->buf->file_pos + size + ngx_pagesize - 1)
-                               & ~((off_t) ngx_pagesize - 1);
-
-                    if (aligned <= cl->buf->file_last) {
-                        size = aligned - cl->buf->file_pos;
-                    }
-                }
+            file_size = (size_t) ngx_chain_coalesce_file(&cl, limit - send);
 
-                file_size += (size_t) size;
-                send += size;
-                fprev = cl->buf->file_pos + size;
-                cl = cl->next;
-
-            } while (cl
-                     && cl->buf->in_file
-                     && send < limit
-                     && file->file->fd == cl->buf->file->fd
-                     && fprev == cl->buf->file_pos);
-        }
-
-        if (file) {
+            send += file_size;
 #if 1
             if (file_size == 0) {
                 ngx_debug_point();
                 return NGX_CHAIN_ERROR;
             }
 #endif
-#if (NGX_HAVE_SENDFILE64)
-            offset = file->file_pos;
-#else
-            offset = (int32_t) file->file_pos;
-#endif
-
-            ngx_log_debug2(NGX_LOG_DEBUG_EVENT, c->log, 0,
-                           "sendfile: @%O %uz", file->file_pos, file_size);
 
-            rc = sendfile(c->fd, file->file->fd, &offset, file_size);
+#if (NGX_THREADS)
+            if (file->file->thread_handler) {
+                rc = ngx_linux_sendfile_thread(c, file, file_size, &sent);
 
-            if (rc == -1) {
-                err = ngx_errno;
+                switch (rc) {
+                case NGX_OK:
+                    thread_handled = 1;
+                    break;
 
-                switch (err) {
-                case NGX_EAGAIN:
+                case NGX_DONE:
+                    thread_complete = 1;
                     break;
 
-                case NGX_EINTR:
-                    eintr = 1;
+                case NGX_AGAIN:
                     break;
 
-                default:
-                    wev->error = 1;
-                    ngx_connection_error(c, err, "sendfile() failed");
+                default: /* NGX_ERROR */
                     return NGX_CHAIN_ERROR;
                 }
 
-                ngx_log_debug0(NGX_LOG_DEBUG_EVENT, c->log, err,
-                               "sendfile() is not ready");
-            }
+            } else
+#endif
+            {
+                n = ngx_linux_sendfile(c, file, file_size);
 
-            sent = rc > 0 ? rc : 0;
+                if (n == NGX_ERROR) {
+                    return NGX_CHAIN_ERROR;
+                }
 
-            ngx_log_debug4(NGX_LOG_DEBUG_EVENT, c->log, 0,
-                           "sendfile: %d, @%O %O:%uz",
-                           rc, file->file_pos, sent, file_size);
+                sent = (n == NGX_AGAIN) ? 0 : n;
+            }
 
         } else {
-            rc = writev(c->fd, header.elts, header.nelts);
+            n = ngx_writev(c, &header);
 
-            if (rc == -1) {
-                err = ngx_errno;
+            if (n == NGX_ERROR) {
+                return NGX_CHAIN_ERROR;
+            }
 
-                switch (err) {
-                case NGX_EAGAIN:
-                    break;
+            sent = (n == NGX_AGAIN) ? 0 : n;
+        }
 
-                case NGX_EINTR:
-                    eintr = 1;
-                    break;
+        c->sent += sent;
 
-                default:
-                    wev->error = 1;
-                    ngx_connection_error(c, err, "writev() failed");
-                    return NGX_CHAIN_ERROR;
-                }
+        in = ngx_chain_update_sent(in, sent);
 
-                ngx_log_debug0(NGX_LOG_DEBUG_EVENT, c->log, err,
-                               "writev() not ready");
+        if ((size_t) (send - prev_send) != sent) {
+#if (NGX_THREADS)
+            if (thread_handled) {
+                return in;
             }
 
-            sent = rc > 0 ? rc : 0;
-
-            ngx_log_debug1(NGX_LOG_DEBUG_EVENT, c->log, 0, "writev: %O", sent);
+            if (thread_complete) {
+                send = prev_send + sent;
+                continue;
+            }
+#endif
+            wev->ready = 0;
+            return in;
         }
 
-        if (send - prev_send == sent) {
-            complete = 1;
+        if (send >= limit || in == NULL) {
+            return in;
         }
+    }
+}
 
-        c->sent += sent;
 
-        for ( /* void */ ; in; in = in->next) {
+static ssize_t
+ngx_linux_sendfile(ngx_connection_t *c, ngx_buf_t *file, size_t size)
+{
+#if (NGX_HAVE_SENDFILE64)
+    off_t      offset;
+#else
+    int32_t    offset;
+#endif
+    ssize_t    n;
+    ngx_err_t  err;
 
-            if (ngx_buf_special(in->buf)) {
-                continue;
-            }
+#if (NGX_HAVE_SENDFILE64)
+    offset = file->file_pos;
+#else
+    offset = (int32_t) file->file_pos;
+#endif
 
-            if (sent == 0) {
-                break;
-            }
+eintr:
 
-            size = ngx_buf_size(in->buf);
+    ngx_log_debug2(NGX_LOG_DEBUG_EVENT, c->log, 0,
+                   "sendfile: @%O %uz", file->file_pos, size);
 
-            if (sent >= size) {
-                sent -= size;
+    n = sendfile(c->fd, file->file->fd, &offset, size);
 
-                if (ngx_buf_in_memory(in->buf)) {
-                    in->buf->pos = in->buf->last;
-                }
+    if (n == -1) {
+        err = ngx_errno;
 
-                if (in->buf->in_file) {
-                    in->buf->file_pos = in->buf->file_last;
-                }
+        switch (err) {
+        case NGX_EAGAIN:
+            ngx_log_debug0(NGX_LOG_DEBUG_EVENT, c->log, err,
+                           "sendfile() is not ready");
+            return NGX_AGAIN;
 
-                continue;
-            }
+        case NGX_EINTR:
+            ngx_log_debug0(NGX_LOG_DEBUG_EVENT, c->log, err,
+                           "sendfile() was interrupted");
+            goto eintr;
 
-            if (ngx_buf_in_memory(in->buf)) {
-                in->buf->pos += (size_t) sent;
-            }
+        default:
+            c->write->error = 1;
+            ngx_connection_error(c, err, "sendfile() failed");
+            return NGX_ERROR;
+        }
+    }
 
-            if (in->buf->in_file) {
-                in->buf->file_pos += sent;
-            }
+    ngx_log_debug3(NGX_LOG_DEBUG_EVENT, c->log, 0, "sendfile: %z of %uz @%O",
+                   n, size, file->file_pos);
 
-            break;
-        }
+    return n;
+}
+
+
+#if (NGX_THREADS)
+
+typedef struct {
+    ngx_buf_t     *file;
+    ngx_socket_t   socket;
+    size_t         size;
+
+    size_t         sent;
+    ngx_err_t      err;
+} ngx_linux_sendfile_ctx_t;
 
-        if (eintr) {
-            continue;
+
+static ngx_int_t
+ngx_linux_sendfile_thread(ngx_connection_t *c, ngx_buf_t *file, size_t size,
+    size_t *sent)
+{
+    ngx_uint_t                 flags;
+    ngx_event_t               *wev;
+    ngx_thread_task_t         *task;
+    ngx_linux_sendfile_ctx_t  *ctx;
+
+    ngx_log_debug3(NGX_LOG_DEBUG_CORE, c->log, 0,
+                   "linux sendfile thread: %d, %uz, %O",
+                   file->file->fd, size, file->file_pos);
+
+    task = c->sendfile_task;
+
+    if (task == NULL) {
+        task = ngx_thread_task_alloc(c->pool, sizeof(ngx_linux_sendfile_ctx_t));
+        if (task == NULL) {
+            return NGX_ERROR;
         }
 
-        if (!complete) {
-            wev->ready = 0;
-            return in;
+        task->handler = ngx_linux_sendfile_thread_handler;
+
+        c->sendfile_task = task;
+    }
+
+    ctx = task->ctx;
+    wev = c->write;
+
+    if (task->event.complete) {
+        task->event.complete = 0;
+
+        if (ctx->err && ctx->err != NGX_EAGAIN) {
+            wev->error = 1;
+            ngx_connection_error(c, ctx->err, "sendfile() failed");
+            return NGX_ERROR;
         }
 
-        if (send >= limit || in == NULL) {
-            return in;
+        *sent = ctx->sent;
+
+        return (ctx->sent == ctx->size) ? NGX_DONE : NGX_AGAIN;
+    }
+
+    ctx->file = file;
+    ctx->socket = c->fd;
+    ctx->size = size;
+
+    if (wev->active) {
+        flags = (ngx_event_flags & NGX_USE_CLEAR_EVENT) ? NGX_CLEAR_EVENT
+                                                        : NGX_LEVEL_EVENT;
+
+        if (ngx_del_event(wev, NGX_WRITE_EVENT, flags) == NGX_ERROR) {
+            return NGX_ERROR;
         }
     }
+
+    if (file->file->thread_handler(task, file->file) != NGX_OK) {
+        return NGX_ERROR;
+    }
+
+    *sent = 0;
+
+    return NGX_OK;
+}
+
+
+static void
+ngx_linux_sendfile_thread_handler(void *data, ngx_log_t *log)
+{
+    ngx_linux_sendfile_ctx_t *ctx = data;
+
+    off_t       offset;
+    ssize_t     n;
+    ngx_buf_t  *file;
+
+    ngx_log_debug0(NGX_LOG_DEBUG_CORE, log, 0, "linux sendfile thread handler");
+
+    file = ctx->file;
+    offset = file->file_pos;
+
+again:
+
+    n = sendfile(ctx->socket, file->file->fd, &offset, ctx->size);
+
+    if (n == -1) {
+        ctx->err = ngx_errno;
+
+    } else {
+        ctx->sent = n;
+        ctx->err = 0;
+    }
+
+#if 0
+    ngx_time_update();
+#endif
+
+    ngx_log_debug4(NGX_LOG_DEBUG_EVENT, log, 0,
+                   "sendfile: %z (err: %i) of %uz @%O",
+                   n, ctx->err, ctx->size, file->file_pos);
+
+    if (ctx->err == NGX_EINTR) {
+        goto again;
+    }
 }
+
+#endif /* NGX_THREADS */