jruby-http-kit 0.0.1

data/src/org/httpkit/server/Frame.java

@@ -0,0 +1,73 @@
+package org.httpkit.server;
+
+import org.httpkit.HttpUtils;
+
+import java.nio.ByteBuffer;
+
+public abstract class Frame {
+    public final byte[] data;
+
+    public Frame(byte data[]) {
+        this.data = data;
+    }
+
+    public static class BinaryFrame extends Frame {
+        public BinaryFrame(byte[] data) {
+            super(data);
+        }
+    }
+
+    public static class CloseFrame extends Frame {
+
+        public final static short CLOSE_NORMAL = 1000;
+        // indicates that an endpoint is "going away", such as a server going down
+        // or a browser having navigated away from a page.
+        public final static short CLOSE_AWAY = 1001;
+        //        public final static short CLOSE_PROTOCOL_ERROR = 1002;
+//        public final static short CLOSE_NOT_IMPL = 1003;
+        // This is a generic status code that can be returned when there is no other
+        // more suitable status code (e.g., 1003 or 1009)
+//        public final static short CLOSE_VIOLATES_POLICY = 1008;
+        public final static short CLOSE_MESG_BIG = 1009;
+//        public final static short CLOSE_SEVER_ERROR = 1011;
+
+//        private static byte[] bytes(short code) {
+//            return ByteBuffer.allocate(2).putShort(code).array();
+//        }
+
+//        public static final CloseFrame NORMAL = new CloseFrame(bytes(CLOSE_NORMAL));
+//        public static final CloseFrame AWAY = new CloseFrame(bytes(CLOSE_AWAY));
+//        public static final CloseFrame MESG_BIG = new CloseFrame(bytes(CLOSE_MESG_BIG));
+//        public static final CloseFrame SERVER_ERROR = new CloseFrame(bytes(CLOSE_MESG_BIG));
+
+        public CloseFrame(byte[] data) {
+            super(data);
+        }
+
+        public int getStatus() {
+            if (data.length >= 2) {
+                return ByteBuffer.wrap(data, 0, 2).getShort();
+            }
+            return CLOSE_NORMAL;
+        }
+    }
+
+    public static class PingFrame extends Frame {
+        public PingFrame(byte[] data) {
+            super(data);
+        }
+    }
+
+    public static class TextFrame extends Frame {
+        private final String msg;
+
+        public TextFrame(byte[] data) {
+            super(data);
+            this.msg = new String(data, HttpUtils.UTF_8);
+        }
+
+        public String getText() {
+            return msg;
+        }
+    }
+}

data/src/org/httpkit/server/HttpAtta.java

@@ -0,0 +1,10 @@
+package org.httpkit.server;
+
+public class HttpAtta extends ServerAtta {
+
+    public HttpAtta(int maxBody, int maxLine) {
+        decoder = new HttpDecoder(maxBody, maxLine);
+    }
+
+    public final HttpDecoder decoder;
+}

data/src/org/httpkit/server/HttpDecoder.java

@@ -0,0 +1,202 @@
+package org.httpkit.server;
+
+import org.httpkit.*;
+
+import java.nio.ByteBuffer;
+import java.util.Arrays;
+import java.util.Map;
+import java.util.TreeMap;
+
+import static org.httpkit.HttpUtils.*;
+import static org.httpkit.HttpVersion.HTTP_1_0;
+import static org.httpkit.HttpVersion.HTTP_1_1;
+
+public class HttpDecoder {
+
+    public enum State {
+        ALL_READ, READ_INITIAL, READ_HEADER, READ_FIXED_LENGTH_CONTENT, READ_CHUNK_SIZE, READ_CHUNKED_CONTENT, READ_CHUNK_FOOTER, READ_CHUNK_DELIMITER,
+    }
+
+    private State state = State.READ_INITIAL;
+    private int readRemaining = 0; // bytes need read
+    private int readCount = 0; // already read bytes count
+
+    HttpRequest request; // package visible
+    private Map<String, Object> headers = new TreeMap<String, Object>();
+    byte[] content;
+
+    private final int maxBody;
+    private final LineReader lineReader;
+
+    public HttpDecoder(int maxBody, int maxLine) {
+        this.maxBody = maxBody;
+        this.lineReader = new LineReader(maxLine);
+    }
+
+    private void createRequest(String sb) throws ProtocolException {
+        int aStart;
+        int aEnd;
+        int bStart;
+        int bEnd;
+        int cStart;
+        int cEnd;
+
+        aStart = findNonWhitespace(sb, 0);
+        aEnd = findWhitespace(sb, aStart);
+
+        bStart = findNonWhitespace(sb, aEnd);
+        bEnd = findWhitespace(sb, bStart);
+
+        cStart = findNonWhitespace(sb, bEnd);
+        cEnd = findEndOfString(sb);
+
+        if (cStart < cEnd) {
+            try {
+                HttpMethod method = HttpMethod.valueOf(sb.substring(aStart, aEnd).toUpperCase());
+                HttpVersion version = HTTP_1_1;
+                if ("HTTP/1.0".equals(sb.substring(cStart, cEnd))) {
+                    version = HTTP_1_0;
+                }
+                request = new HttpRequest(method, sb.substring(bStart, bEnd), version);
+            } catch (Exception e) {
+                throw new ProtocolException("method not understand");
+            }
+        } else {
+            throw new ProtocolException("not http?");
+        }
+    }
+
+    public HttpRequest decode(ByteBuffer buffer) throws LineTooLargeException,
+            ProtocolException, RequestTooLargeException {
+        String line;
+        while (buffer.hasRemaining()) {
+            switch (state) {
+                case ALL_READ:
+                    return request;
+                case READ_INITIAL:
+                    line = lineReader.readLine(buffer);
+                    if (line != null) {
+                        createRequest(line);
+                        state = State.READ_HEADER;
+                    }
+                    break;
+                case READ_HEADER:
+                    readHeaders(buffer);
+                    break;
+                case READ_CHUNK_SIZE:
+                    line = lineReader.readLine(buffer);
+                    if (line != null) {
+                        readRemaining = getChunkSize(line);
+                        if (readRemaining == 0) {
+                            state = State.READ_CHUNK_FOOTER;
+                        } else {
+                            throwIfBodyIsTooLarge();
+                            if (content == null) {
+                                content = new byte[readRemaining];
+                            } else if (content.length < readCount + readRemaining) {
+                                // *1.3 to protect slow client
+                                int newLength = (int) ((readRemaining + readCount) * 1.3);
+                                content = Arrays.copyOf(content, newLength);
+                            }
+                            state = State.READ_CHUNKED_CONTENT;
+                        }
+                    }
+                    break;
+                case READ_FIXED_LENGTH_CONTENT:
+                    readFixedLength(buffer);
+                    if (readRemaining == 0) {
+                        finish();
+                    }
+                    break;
+                case READ_CHUNKED_CONTENT:
+                    readFixedLength(buffer);
+                    if (readRemaining == 0) {
+                        state = State.READ_CHUNK_DELIMITER;
+                    }
+                    break;
+                case READ_CHUNK_FOOTER:
+                    readEmptyLine(buffer);
+                    finish();
+                    break;
+                case READ_CHUNK_DELIMITER:
+                    readEmptyLine(buffer);
+                    state = State.READ_CHUNK_SIZE;
+                    break;
+            }
+        }
+        return state == State.ALL_READ ? request : null;
+    }
+
+    private void finish() {
+        state = State.ALL_READ;
+        request.setBody(content, readCount);
+    }
+
+    void readEmptyLine(ByteBuffer buffer) {
+        byte b = buffer.get();
+        if (b == CR && buffer.hasRemaining()) {
+            buffer.get(); // should be LF
+        }
+    }
+
+    void readFixedLength(ByteBuffer buffer) {
+        int toRead = Math.min(buffer.remaining(), readRemaining);
+        buffer.get(content, readCount, toRead);
+        readRemaining -= toRead;
+        readCount += toRead;
+    }
+
+    private void readHeaders(ByteBuffer buffer) throws LineTooLargeException,
+            RequestTooLargeException, ProtocolException {
+        String line = lineReader.readLine(buffer);
+        while (line != null && !line.isEmpty()) {
+            HttpUtils.splitAndAddHeader(line, headers);
+            line = lineReader.readLine(buffer);
+        }
+
+        if (line == null) {
+            return;
+        }
+
+        request.setHeaders(headers);
+
+        String te = HttpUtils.getStringValue(headers, TRANSFER_ENCODING);
+        if (CHUNKED.equals(te)) {
+            state = State.READ_CHUNK_SIZE;
+        } else {
+            String cl = HttpUtils.getStringValue(headers, CONTENT_LENGTH);
+            if (cl != null) {
+                try {
+                    readRemaining = Integer.parseInt(cl);
+                    if (readRemaining > 0) {
+                        throwIfBodyIsTooLarge();
+                        content = new byte[readRemaining];
+                        state = State.READ_FIXED_LENGTH_CONTENT;
+                    } else {
+                        state = State.ALL_READ;
+                    }
+                } catch (NumberFormatException e) {
+                    throw new ProtocolException(e.getMessage());
+                }
+            } else {
+                state = State.ALL_READ;
+            }
+        }
+    }
+
+    public void reset() {
+        state = State.READ_INITIAL;
+        headers = new TreeMap<String, Object>();
+        readCount = 0;
+        content = null;
+        lineReader.reset();
+        request = null;
+    }
+
+    private void throwIfBodyIsTooLarge() throws RequestTooLargeException {
+        if (readCount + readRemaining > maxBody) {
+            throw new RequestTooLargeException("request body " + (readCount + readRemaining)
+                    + "; max request body " + maxBody);
+        }
+    }
+}

data/src/org/httpkit/server/HttpRequest.java

@@ -0,0 +1,109 @@
+package org.httpkit.server;
+
+import org.httpkit.*;
+
+import java.io.InputStream;
+import java.net.InetSocketAddress;
+import java.util.Map;
+
+import static org.httpkit.HttpUtils.*;
+import static org.httpkit.HttpVersion.HTTP_1_1;
+
+public class HttpRequest {
+    public final String queryString;
+    public final String uri;
+    public final HttpMethod method;
+    public final HttpVersion version;
+
+    private byte[] body;
+
+    public int serverPort = 80;
+    public String serverName;
+    public Map<String, Object> headers;
+    public int contentLength = 0;
+    public String contentType;
+    public String charset = "utf8";
+    public boolean isKeepAlive = false;
+    public boolean isWebSocket = false;
+
+    public InetSocketAddress remoteAddr;
+    public AsyncChannel channel;
+
+    public HttpRequest(HttpMethod method, String url, HttpVersion version) {
+        this.method = method;
+        this.version = version;
+        int idx = url.indexOf('?');
+        if (idx > 0) {
+            uri = url.substring(0, idx);
+            queryString = url.substring(idx + 1);
+        } else {
+            uri = url;
+            queryString = null;
+        }
+    }
+
+    public InputStream getBody() {
+        if (body != null) {
+            return new BytesInputStream(body, contentLength);
+        }
+        return null;
+    }
+
+    public String getRemoteAddr() {
+        String h = getStringValue(headers, HttpUtils.X_FORWARDED_FOR);
+        if (null != h) {
+            int idx = h.indexOf(',');
+            if (idx == -1) {
+                return h;
+            } else {
+                // X-Forwarded-For: client, proxy1, proxy2
+                return h.substring(0, idx);
+            }
+        } else {
+            return remoteAddr.getAddress().getHostAddress();
+        }
+    }
+
+    public void setBody(byte[] body, int count) {
+        this.body = body;
+        this.contentLength = count;
+    }
+
+    public void setHeaders(Map<String, Object> headers) {
+        String h = getStringValue(headers, "host");
+        if (h != null) {
+            int idx = h.lastIndexOf(':');
+            if (idx != -1) {
+                this.serverName = h.substring(0, idx);
+                serverPort = Integer.valueOf(h.substring(idx + 1));
+            } else {
+                this.serverName = h;
+            }
+        }
+
+        String ct = getStringValue(headers, CONTENT_TYPE);
+        if (ct != null) {
+            int idx = ct.indexOf(";");
+            if (idx != -1) {
+                int cidx = ct.indexOf(CHARSET, idx);
+                if (cidx != -1) {
+                    contentType = ct.substring(0, idx);
+                    charset = ct.substring(cidx + CHARSET.length());
+                } else {
+                    contentType = ct;
+                }
+            } else {
+                contentType = ct;
+            }
+        }
+
+        String con = getStringValue(headers, CONNECTION);
+        if (con != null) {
+            con = con.toLowerCase();
+        }
+
+        isKeepAlive = (version == HTTP_1_1 && !"close".equals(con)) || "keep-alive".equals(con);
+        isWebSocket = "websocket".equalsIgnoreCase(getStringValue(headers, "upgrade"));
+        this.headers = headers;
+    }
+}

data/src/org/httpkit/server/HttpServer.java

@@ -0,0 +1,281 @@
+package org.httpkit.server;
+
+import org.httpkit.*;
+
+import java.io.IOException;
+import java.net.InetSocketAddress;
+import java.nio.ByteBuffer;
+import java.nio.channels.*;
+import java.util.*;
+import java.util.concurrent.ConcurrentLinkedQueue;
+
+import static java.nio.channels.SelectionKey.*;
+import static org.httpkit.HttpUtils.HttpEncode;
+import static org.httpkit.HttpUtils.WsEncode;
+import static org.httpkit.server.Frame.CloseFrame.*;
+
+public class HttpServer implements Runnable {
+
+    static final String THREAD_NAME = "server-loop";
+
+    private final IHandler handler;
+    private final int maxBody;
+    private final int maxLine;
+
+    private final Selector selector;
+    private final ServerSocketChannel serverChannel;
+
+    private Thread serverThread;
+
+    private final ConcurrentLinkedQueue<SelectionKey> pending = new ConcurrentLinkedQueue<SelectionKey>();
+    // shared, single thread
+    private final ByteBuffer buffer = ByteBuffer.allocateDirect(1024 * 64);
+
+    public HttpServer(String ip, int port, IHandler handler, int maxBody, int maxLine)
+            throws IOException {
+        this.handler = handler;
+        this.maxLine = maxLine;
+        this.maxBody = maxBody;
+        this.selector = Selector.open();
+        this.serverChannel = ServerSocketChannel.open();
+        serverChannel.configureBlocking(false);
+        serverChannel.socket().bind(new InetSocketAddress(ip, port));
+        serverChannel.register(selector, OP_ACCEPT);
+    }
+
+    void accept(SelectionKey key) {
+        ServerSocketChannel ch = (ServerSocketChannel) key.channel();
+        SocketChannel s;
+        try {
+            while ((s = ch.accept()) != null) {
+                s.configureBlocking(false);
+                HttpAtta atta = new HttpAtta(maxBody, maxLine);
+                SelectionKey k = s.register(selector, OP_READ, atta);
+                atta.channel = new AsyncChannel(k, this);
+            }
+        } catch (Exception e) {
+            // too many open files. do not quit
+            HttpUtils.printError("accept incoming request", e);
+        }
+    }
+
+    private void closeKey(final SelectionKey key, int status) {
+        try {
+            key.channel().close();
+        } catch (Exception ignore) {
+        }
+
+        ServerAtta att = (ServerAtta) key.attachment();
+        if (att instanceof HttpAtta) {
+            handler.clientClose(att.channel, -1);
+        } else {
+            handler.clientClose(att.channel, status);
+        }
+    }
+
+    private void decodeHttp(HttpAtta atta, SelectionKey key, SocketChannel ch) {
+        try {
+            do {
+                AsyncChannel channel = atta.channel;
+                HttpRequest request = atta.decoder.decode(buffer);
+                if (request != null) {
+                    channel.reset(request);
+                    if (request.isWebSocket) {
+                        key.attach(new WsAtta(channel));
+                    } else {
+                        atta.keepalive = request.isKeepAlive;
+                    }
+                    request.channel = channel;
+                    request.remoteAddr = (InetSocketAddress) ch.socket().getRemoteSocketAddress();
+                    handler.handle(request, new RespCallback(key, this));
+                    // pipelining not supported : need queue to ensure order
+                    atta.decoder.reset();
+                }
+            } while (buffer.hasRemaining()); // consume all
+        } catch (ProtocolException e) {
+            closeKey(key, -1);
+        } catch (RequestTooLargeException e) {
+            atta.keepalive = false;
+            tryWrite(key, HttpEncode(413, new HeaderMap(), e.getMessage()));
+        } catch (LineTooLargeException e) {
+            atta.keepalive = false; // close after write
+            tryWrite(key, HttpEncode(414, new HeaderMap(), e.getMessage()));
+        }
+    }
+
+    private void decodeWs(WsAtta atta, SelectionKey key) {
+        try {
+            do {
+                Frame frame = atta.decoder.decode(buffer);
+                if (frame instanceof TextFrame || frame instanceof BinaryFrame) {
+                    handler.handle(atta.channel, frame);
+                    atta.decoder.reset();
+                } else if (frame instanceof PingFrame) {
+                    atta.decoder.reset();
+                    tryWrite(key, WsEncode(WSDecoder.OPCODE_PONG, frame.data));
+                } else if (frame instanceof CloseFrame) {
+                    handler.clientClose(atta.channel, ((CloseFrame) frame).getStatus());
+                    // close the TCP connection after sent
+                    atta.keepalive = false;
+                    tryWrite(key, WsEncode(WSDecoder.OPCODE_CLOSE, frame.data));
+                }
+            } while (buffer.hasRemaining()); // consume all
+        } catch (ProtocolException e) {
+            System.err.printf("%s [%s] WARN - %s\n", new Date(), THREAD_NAME, e.getMessage());
+            closeKey(key, CLOSE_MESG_BIG); // TODO more specific error
+        }
+    }
+
+    private void doRead(final SelectionKey key) {
+        SocketChannel ch = (SocketChannel) key.channel();
+        try {
+            buffer.clear(); // clear for read
+            int read = ch.read(buffer);
+            if (read == -1) {
+                // remote entity shut the socket down cleanly.
+                closeKey(key, CLOSE_AWAY);
+            } else if (read > 0) {
+                buffer.flip(); // flip for read
+                final ServerAtta atta = (ServerAtta) key.attachment();
+                if (atta instanceof HttpAtta) {
+                    decodeHttp((HttpAtta) atta, key, ch);
+                } else {
+                    decodeWs((WsAtta) atta, key);
+                }
+            }
+        } catch (IOException e) { // the remote forcibly closed the connection
+            closeKey(key, CLOSE_AWAY);
+        }
+    }
+
+    private void doWrite(SelectionKey key) {
+        ServerAtta atta = (ServerAtta) key.attachment();
+        SocketChannel ch = (SocketChannel) key.channel();
+        try {
+            // the sync is per socket (per client). virtually, no contention
+            // 1. keep byte data order, 2. ensure visibility
+            synchronized (atta) {
+                LinkedList<ByteBuffer> toWrites = atta.toWrites;
+                int size = toWrites.size();
+                if (size == 1) {
+                    ch.write(toWrites.get(0));
+                    // TODO investigate why needed.
+                    // ws request for write, but has no data?
+                } else if (size > 0) {
+                    ByteBuffer buffers[] = new ByteBuffer[size];
+                    toWrites.toArray(buffers);
+                    ch.write(buffers, 0, buffers.length);
+                }
+                Iterator<ByteBuffer> ite = toWrites.iterator();
+                while (ite.hasNext()) {
+                    if (!ite.next().hasRemaining()) {
+                        ite.remove();
+                    }
+                }
+                // all done
+                if (toWrites.size() == 0) {
+                    if (atta.isKeepAlive()) {
+                        key.interestOps(OP_READ);
+                    } else {
+                        closeKey(key, CLOSE_NORMAL);
+                    }
+                }
+            }
+        } catch (IOException e) { // the remote forcibly closed the connection
+            closeKey(key, CLOSE_AWAY);
+        }
+    }
+
+    public void tryWrite(final SelectionKey key, ByteBuffer... buffers) {
+        ServerAtta atta = (ServerAtta) key.attachment();
+        synchronized (atta) {
+            if (atta.toWrites.isEmpty()) {
+                SocketChannel ch = (SocketChannel) key.channel();
+                try {
+                    // TCP buffer most of time is empty, writable(8K ~ 256k)
+                    // One IO thread => One thread reading + Many thread writing
+                    // Save 2 system call
+                    ch.write(buffers, 0, buffers.length);
+                    if (buffers[buffers.length - 1].hasRemaining()) {
+                        for (ByteBuffer b : buffers) {
+                            if (b.hasRemaining()) {
+                                atta.toWrites.add(b);
+                            }
+                        }
+                        pending.add(key);
+                        selector.wakeup();
+                    } else if (!atta.isKeepAlive()) {
+                        closeKey(key, CLOSE_NORMAL);
+                    }
+                } catch (IOException e) {
+                    closeKey(key, CLOSE_AWAY);
+                }
+            } else {
+                // If has pending write, order should be maintained. (WebSocket)
+                Collections.addAll(atta.toWrites, buffers);
+                pending.add(key);
+                selector.wakeup();
+            }
+        }
+    }
+
+    public void run() {
+        while (true) {
+            try {
+                SelectionKey k = null;
+                while ((k = pending.poll()) != null) {
+                    if (k.isValid()) {
+                        k.interestOps(OP_WRITE);
+                    }
+                }
+                if (selector.select() <= 0) {
+                    continue;
+                }
+                Set<SelectionKey> selectedKeys = selector.selectedKeys();
+                for (SelectionKey key : selectedKeys) {
+                    // TODO I do not know if this is needed
+                    // if !valid, isAcceptable, isReadable.. will Exception
+                    // run hours happily after commented, but not sure.
+                    if (!key.isValid()) {
+                        continue;
+                    }
+                    if (key.isAcceptable()) {
+                        accept(key);
+                    } else if (key.isReadable()) {
+                        doRead(key);
+                    } else if (key.isWritable()) {
+                        doWrite(key);
+                    }
+                }
+                selectedKeys.clear();
+            } catch (ClosedSelectorException ignore) {
+                return; // stopped
+                // do not exits the while IO event loop. if exits, then will not process any IO event
+                // jvm can catch any exception, including OOM
+            } catch (Throwable e) { // catch any exception(including OOM), print it
+                HttpUtils.printError("http server loop error, should not happen", e);
+            }
+        }
+    }
+
+    public void start() throws IOException {
+        serverThread = new Thread(this, THREAD_NAME);
+        serverThread.start();
+    }
+
+    public void stop() {
+        if (selector.isOpen()) {
+            try {
+                serverChannel.close();
+                Set<SelectionKey> keys = selector.keys();
+                for (SelectionKey k : keys) {
+                    k.channel().close();
+                }
+                selector.close();
+                handler.close();
+            } catch (IOException ignore) {
+            }
+            serverThread.interrupt();
+        }
+    }
+}