jruby-http-kit 0.0.1

data/src/org/httpkit/client/HttpClient.java

@@ -0,0 +1,393 @@
+package org.httpkit.client;
+
+import org.httpkit.*;
+import org.httpkit.ProtocolException;
+
+import javax.net.ssl.SSLContext;
+import javax.net.ssl.SSLEngine;
+import javax.net.ssl.SSLException;
+import java.io.IOException;
+import java.net.*;
+import java.nio.ByteBuffer;
+import java.nio.channels.SelectionKey;
+import java.nio.channels.Selector;
+import java.nio.channels.SocketChannel;
+import java.security.NoSuchAlgorithmException;
+import java.util.Iterator;
+import java.util.Queue;
+import java.util.Set;
+import java.util.concurrent.ConcurrentLinkedQueue;
+import java.util.concurrent.atomic.AtomicInteger;
+
+import static java.lang.System.currentTimeMillis;
+import static java.nio.channels.SelectionKey.*;
+import static org.httpkit.HttpUtils.SP;
+import static org.httpkit.HttpUtils.getServerAddr;
+import static org.httpkit.client.State.ALL_READ;
+import static org.httpkit.client.State.READ_INITIAL;
+
+public final class HttpClient implements Runnable {
+    private static final AtomicInteger ID = new AtomicInteger(0);
+
+    public static final SSLContext DEFAULT_CONTEXT;
+
+    static {
+        try {
+            DEFAULT_CONTEXT = SSLContext.getDefault();
+        } catch (NoSuchAlgorithmException e) {
+            throw new Error("Failed to initialize SSLContext", e);
+        }
+    }
+
+    private final Queue<Request> pending = new ConcurrentLinkedQueue<Request>();
+    private final PriorityQueue<Request> requests = new PriorityQueue<Request>();
+    private final PriorityQueue<PersistentConn> keepalives = new PriorityQueue<PersistentConn>();
+
+    private volatile boolean running = true;
+
+    // shared, single thread
+    private final ByteBuffer buffer = ByteBuffer.allocateDirect(1024 * 64);
+    private final Selector selector;
+
+    public HttpClient() throws IOException {
+        int id = ID.incrementAndGet();
+        String name = "client-loop";
+        if (id > 1) {
+            name = name + "#" + id;
+        }
+        selector = Selector.open();
+        Thread t = new Thread(this, name);
+        t.setDaemon(true);
+        t.start();
+    }
+
+    private void clearTimeout(long now) {
+        Request r;
+        while ((r = requests.peek()) != null) {
+            if (r.isTimeout(now)) {
+                String msg = "connect timeout: ";
+                if (r.isConnected) {
+                    msg = "read timeout: ";
+                }
+                // will remove it from queue
+                r.finish(new TimeoutException(msg + r.cfg.timeout + "ms"));
+                if (r.key != null) {
+                    closeQuietly(r.key);
+                }
+            } else {
+                break;
+            }
+        }
+
+        PersistentConn pc;
+        while ((pc = keepalives.peek()) != null) {
+            if (pc.isTimeout(now)) {
+                closeQuietly(pc.key);
+                keepalives.poll();
+            } else {
+                break;
+            }
+        }
+    }
+
+    /**
+     * http-kit think all connections are keep-alived (since some say it is, but
+     * actually is not). but, some are not, http-kit pick them out after the fact
+     * <ol>
+     * <li>The connection is reused</li>
+     * <li>No data received</li>
+     * </ol>
+     */
+    private boolean cleanAndRetryIfBroken(SelectionKey key, Request req) {
+        closeQuietly(key);
+        keepalives.remove(key);
+        // keep-alived connection, remote server close it without sending byte
+        if (req.isReuseConn && req.decoder.state == READ_INITIAL) {
+            for (ByteBuffer b : req.request) {
+                b.position(0); // reset for retry
+            }
+            req.isReuseConn = false;
+            requests.remove(req); // remove from timeout queue
+            pending.offer(req); // queue for retry
+            selector.wakeup();
+            return true; // retry: re-open a connection to server, sent the request again
+        }
+        return false;
+    }
+
+    private void doRead(SelectionKey key, long now) {
+        Request req = (Request) key.attachment();
+        SocketChannel ch = (SocketChannel) key.channel();
+        int read = 0;
+        try {
+            buffer.clear();
+            if (req instanceof HttpsRequest) {
+                HttpsRequest httpsReq = (HttpsRequest) req;
+                if (httpsReq.handshaken) {
+                    // SSLEngine closed => fine, will return -1 in the next run
+                    read = httpsReq.unwrapRead(buffer);
+                } else {
+                    read = httpsReq.doHandshake(buffer);
+                }
+            } else {
+                read = ch.read(buffer);
+            }
+        } catch (IOException e) { // The remote forcibly closed the connection
+            if (!cleanAndRetryIfBroken(key, req)) {
+                req.finish(e); // os X get Connection reset by peer error,
+            }
+            // java.security.InvalidAlgorithmParameterException: Prime size must be multiple of 64, and can only range from 512 to 1024 (inclusive)
+            // java.lang.RuntimeException: Could not generate DH keypair
+        } catch (Exception e) {
+            req.finish(e);
+        }
+
+        if (read == -1) { // read all, remote closed it cleanly
+            if (!cleanAndRetryIfBroken(key, req)) {
+                req.finish();
+            }
+        } else if (read > 0) {
+            req.onProgress(now);
+            buffer.flip();
+            try {
+                if (req.decoder.decode(buffer) == ALL_READ) {
+                    req.finish();
+                    if (req.cfg.keepAlive > 0) {
+                        keepalives.offer(new PersistentConn(now + req.cfg.keepAlive, req.addr, key));
+                    } else {
+                        closeQuietly(key);
+                    }
+                }
+            } catch (HTTPException e) {
+                closeQuietly(key);
+                req.finish(e);
+            } catch (Exception e) {
+                closeQuietly(key);
+                req.finish(e);
+                HttpUtils.printError("should not happen", e); // decoding
+            }
+        }
+    }
+
+    private void closeQuietly(SelectionKey key) {
+        try {
+            // TODO engine.closeInbound
+            key.channel().close();
+        } catch (Exception ignore) {
+        }
+    }
+
+    private void doWrite(SelectionKey key) {
+        Request req = (Request) key.attachment();
+        SocketChannel ch = (SocketChannel) key.channel();
+        try {
+            if (req instanceof HttpsRequest) {
+                HttpsRequest httpsReq = (HttpsRequest) req;
+                if (httpsReq.handshaken) {
+                    // will flip to OP_READ
+                    httpsReq.writeWrappedRequest();
+                } else {
+                    buffer.clear();
+                    if (httpsReq.doHandshake(buffer) < 0) {
+                        req.finish(); // will be a No status exception
+                    }
+                }
+            } else {
+                ByteBuffer[] buffers = req.request;
+                ch.write(buffers);
+                if (!buffers[buffers.length - 1].hasRemaining()) {
+                    key.interestOps(OP_READ);
+                }
+            }
+        } catch (IOException e) {
+            if (!cleanAndRetryIfBroken(key, req)) {
+                req.finish(e);
+            }
+        } catch (Exception e) { // rarely happen
+            req.finish(e);
+        }
+    }
+
+    public void exec(String url, RequestConfig cfg, SSLEngine engine, IRespListener cb) {
+        URI uri;
+        try {
+            uri = new URI(url);
+        } catch (URISyntaxException e) {
+            cb.onThrowable(e);
+            return;
+        }
+
+        if (uri.getHost() == null) {
+            cb.onThrowable(new IllegalArgumentException("host is null: " + url));
+            return;
+        }
+
+        String scheme = uri.getScheme();
+        if (!"http".equals(scheme) && !"https".equals(scheme)) {
+            String message = (scheme == null) ? "No protocol specified" : scheme + " is not supported";
+            cb.onThrowable(new ProtocolException(message));
+            return;
+        }
+
+        InetSocketAddress addr;
+        try {
+            addr = getServerAddr(uri);
+        } catch (UnknownHostException e) {
+            cb.onThrowable(e);
+            return;
+        }
+
+        // copy to modify, normalize header
+        HeaderMap headers = HeaderMap.camelCase(cfg.headers);
+        headers.put("Host", HttpUtils.getHost(uri));
+
+        if (!headers.containsKey("Accept")) // allow override
+            headers.put("Accept", "*/*");
+        if (!headers.containsKey("User-Agent")) // allow override
+            headers.put("User-Agent", RequestConfig.DEFAULT_USER_AGENT); // default
+        if (!headers.containsKey("Accept-Encoding"))
+            headers.put("Accept-Encoding", "gzip, deflate"); // compression is good
+
+        ByteBuffer request[];
+        try {
+            request = encode(cfg.method, headers, cfg.body, uri);
+        } catch (IOException e) {
+            cb.onThrowable(e);
+            return;
+        }
+        if ("https".equals(scheme)) {
+            if (engine == null) {
+                engine = DEFAULT_CONTEXT.createSSLEngine();
+            }
+            engine.setUseClientMode(true);
+            pending.offer(new HttpsRequest(addr, request, cb, requests, cfg, engine));
+        } else {
+            pending.offer(new Request(addr, request, cb, requests, cfg));
+        }
+
+//        pending.offer(new Request(addr, request, cb, requests, cfg));
+        selector.wakeup();
+    }
+
+    private ByteBuffer[] encode(HttpMethod method, HeaderMap headers, Object body,
+                                URI uri) throws IOException {
+        ByteBuffer bodyBuffer = HttpUtils.bodyBuffer(body);
+
+        if (body != null) {
+            headers.put("Content-Length", Integer.toString(bodyBuffer.remaining()));
+        } else {
+            headers.put("Content-Length", "0");
+        }
+        DynamicBytes bytes = new DynamicBytes(196);
+        bytes.append(method.toString()).append(SP).append(HttpUtils.getPath(uri));
+        bytes.append(" HTTP/1.1\r\n");
+        headers.encodeHeaders(bytes);
+        ByteBuffer headBuffer = ByteBuffer.wrap(bytes.get(), 0, bytes.length());
+
+        if (bodyBuffer == null) {
+            return new ByteBuffer[]{headBuffer};
+        } else {
+            return new ByteBuffer[]{headBuffer, bodyBuffer};
+        }
+    }
+
+    private void finishConnect(SelectionKey key, long now) {
+        SocketChannel ch = (SocketChannel) key.channel();
+        Request req = (Request) key.attachment();
+        try {
+            if (ch.finishConnect()) {
+                req.isConnected = true;
+                req.onProgress(now);
+                key.interestOps(OP_WRITE);
+                if (req instanceof HttpsRequest) {
+                    ((HttpsRequest) req).engine.beginHandshake();
+                }
+            }
+        } catch (IOException e) {
+            closeQuietly(key); // not added to kee-alive yet;
+            req.finish(e);
+        }
+    }
+
+    private void processPending() {
+        Request job = null;
+        while ((job = pending.poll()) != null) {
+            if (job.cfg.keepAlive > 0) {
+                PersistentConn con = keepalives.remove(job.addr);
+                if (con != null) { // keep alive
+                    SelectionKey key = con.key;
+                    if (key.isValid()) {
+                        job.isReuseConn = true;
+                        // reuse key, engine
+                        try {
+                            job.recycle((Request) key.attachment());
+                            key.attach(job);
+                            key.interestOps(OP_WRITE);
+                            requests.offer(job);
+                            continue;
+                        } catch (SSLException e) {
+                            closeQuietly(key); // https wrap SSLException, start from fresh
+                        }
+                    } else {
+                        // this should not happen often
+                        closeQuietly(key);
+                    }
+                }
+            }
+            try {
+                SocketChannel ch = SocketChannel.open();
+                ch.configureBlocking(false);
+                // saved for timeout
+                job.key = ch.register(selector, OP_CONNECT, job);
+                ch.connect(job.addr);
+                requests.offer(job);
+            } catch (IOException e) {
+                job.finish(e);
+                HttpUtils.printError("Try to connect " + job.addr, e);
+            }
+        }
+    }
+
+    public void run() {
+        while (running) {
+            try {
+                long now = currentTimeMillis();
+                int select = selector.select(2000);
+                if (select > 0) {
+                    Set<SelectionKey> selectedKeys = selector.selectedKeys();
+                    Iterator<SelectionKey> ite = selectedKeys.iterator();
+                    while (ite.hasNext()) {
+                        SelectionKey key = ite.next();
+                        if (!key.isValid()) {
+                            continue;
+                        }
+                        if (key.isConnectable()) {
+                            finishConnect(key, now);
+                        } else if (key.isReadable()) {
+                            doRead(key, now);
+                        } else if (key.isWritable()) {
+                            doWrite(key);
+                        }
+                        ite.remove();
+                    }
+                }
+                clearTimeout(now);
+                processPending();
+            } catch (Throwable e) { // catch any exception (including OOM), print it: do not exits the loop
+                HttpUtils.printError("select exception, should not happen", e);
+            }
+        }
+    }
+
+    public void stop() throws IOException {
+        running = false;
+        if (selector != null) {
+            selector.close();
+        }
+    }
+
+    @Override
+    public String toString() {
+        return this.getClass().getCanonicalName();
+    }
+}

data/src/org/httpkit/client/HttpsRequest.java

@@ -0,0 +1,141 @@
+package org.httpkit.client;
+
+import org.httpkit.PriorityQueue;
+
+import javax.net.ssl.SSLEngine;
+import javax.net.ssl.SSLEngineResult;
+import javax.net.ssl.SSLEngineResult.Status;
+import javax.net.ssl.SSLException;
+import java.io.IOException;
+import java.net.InetSocketAddress;
+import java.nio.ByteBuffer;
+import java.nio.channels.SelectionKey;
+import java.nio.channels.SocketChannel;
+
+public class HttpsRequest extends Request {
+    private static final ByteBuffer EMPTY_BUFFER = ByteBuffer.allocate(0);
+
+    public HttpsRequest(InetSocketAddress addr, ByteBuffer[] request, IRespListener handler,
+                        PriorityQueue<Request> clients, RequestConfig config, SSLEngine engine) {
+        super(addr, request, handler, clients, config);
+        this.engine = engine;
+    }
+
+    SSLEngine engine; // package private
+    private ByteBuffer myNetData = ByteBuffer.allocate(40 * 1024);
+    private ByteBuffer peerNetData = ByteBuffer.allocate(40 * 1024);
+    boolean handshaken = false;
+
+    final int unwrapRead(ByteBuffer peerAppData) throws IOException {
+        // TODO, make sure peerNetData has remaining place
+        int read = ((SocketChannel) key.channel()).read(peerNetData), unwrapped = 0;
+        if (read > 0) {
+            peerNetData.flip();
+            SSLEngineResult res;
+            while ((res = engine.unwrap(peerNetData, peerAppData)).getStatus() == Status.OK) {
+                unwrapped += res.bytesProduced();
+                if (!peerNetData.hasRemaining())
+                    break;
+            }
+            peerNetData.compact();
+            switch (res.getStatus()) {
+                case OK:
+                case BUFFER_UNDERFLOW: // need more data
+                    return unwrapped;
+                case CLOSED:
+                    return unwrapped > 0 ? unwrapped : -1;
+                case BUFFER_OVERFLOW:
+                    return -1; // can't => peerAppData is 64k
+            }
+            return unwrapped;
+        } else {
+            return read;
+        }
+    }
+
+    private void wrapRequest() throws SSLException {
+        myNetData.clear();
+        SSLEngineResult res = engine.wrap(request, myNetData);
+        if (res.getStatus() != Status.OK) {
+            // TODO larger buffer, uberflow?
+        }
+        myNetData.flip();
+    }
+
+    final void writeWrappedRequest() throws IOException {
+        if (myNetData.hasRemaining()) {
+            ((SocketChannel) key.channel()).write(myNetData);
+        } else if (request[request.length - 1].hasRemaining()) {
+            wrapRequest();
+            ((SocketChannel) key.channel()).write(myNetData);
+        }
+        if (myNetData.hasRemaining() || request[request.length - 1].hasRemaining()) {
+            // need more write
+            if ((key.interestOps() & SelectionKey.OP_WRITE) == 0)
+                key.interestOps(SelectionKey.OP_WRITE);
+        } else {
+            // OK, request sent
+            key.interestOps(SelectionKey.OP_READ);
+        }
+    }
+
+    final int doHandshake(ByteBuffer peerAppData) throws IOException {
+        SSLEngineResult.HandshakeStatus hs = engine.getHandshakeStatus();
+        while (!handshaken) {
+            switch (hs) {
+                case NEED_TASK:
+                    Runnable runnable;
+                    while ((runnable = engine.getDelegatedTask()) != null) {
+                        runnable.run();
+                    }
+                    break;
+                case NEED_UNWRAP:
+                    int read = ((SocketChannel) key.channel()).read(peerNetData);
+                    if (read < 0) {
+                        return -1;
+                    } else {
+                        peerNetData.flip();
+                        SSLEngineResult res = engine.unwrap(peerNetData, peerAppData);
+                        peerNetData.compact();
+                        switch (res.getStatus()) {
+                            case BUFFER_OVERFLOW: // Not possible, peerAppData is 64k
+                                break;
+                            case CLOSED:
+                                return -1;
+                            case BUFFER_UNDERFLOW: // need more data from peer
+                                return 0;
+                        }
+                        // do not flip to write here, since TCP buffer is writable
+                    }
+                    break;
+                case NEED_WRAP:
+                    SSLEngineResult res = engine.wrap(EMPTY_BUFFER, myNetData);
+                    myNetData.flip();
+                    ((SocketChannel) key.channel()).write(myNetData);
+                    if (myNetData.hasRemaining()) {
+                        // TODO, make sure data get written
+                    } else {
+                        myNetData.clear();
+                        if (res.getHandshakeStatus() != SSLEngineResult.HandshakeStatus.NEED_WRAP)
+                            key.interestOps(SelectionKey.OP_READ);
+                    }
+                    break;
+            }
+            hs = engine.getHandshakeStatus();
+            handshaken = hs == SSLEngineResult.HandshakeStatus.NOT_HANDSHAKING
+                    || hs == SSLEngineResult.HandshakeStatus.FINISHED;
+            if (handshaken) {
+                wrapRequest();
+                writeWrappedRequest(); // TCP buffer maybe empty this time
+            }
+        }
+        return 0;
+    }
+
+    public void recycle(Request old) throws SSLException {
+        super.recycle(old);
+        this.engine = ((HttpsRequest) old).engine;
+        this.handshaken = true;
+        wrapRequest(); // prepare for write
+    }
+}

data/src/org/httpkit/client/IFilter.java

@@ -0,0 +1,53 @@
+package org.httpkit.client;
+
+import org.httpkit.DynamicBytes;
+
+import java.util.Map;
+
+/**
+ * allow to abort the connection. for example, a crawler may abort the
+ * connection if not text
+ *
+ * @author feng
+ */
+public interface IFilter {
+    public final static IFilter ACCEPT_ALL = new IFilter() {
+        public boolean accept(DynamicBytes partialBody) {
+            return true;
+        }
+
+        public boolean accept(Map<String, Object> headers) {
+            return true;
+        }
+
+        public String toString() {
+            return "Response Filter: ACCEPT all response";
+        }
+    };
+
+    // if the response is too large, protect OOM
+    // For example, HTML expected, but a big mp4 file is returned
+    public static class MaxBodyFilter implements IFilter {
+        private final int length;
+
+        public MaxBodyFilter(int maxLength) {
+            this.length = maxLength;
+        }
+
+        public boolean accept(Map<String, Object> headers) {
+            return true;
+        }
+
+        public String toString() {
+            return "Response Filter: ACCEPT when body's length <= " + length;
+        }
+
+        public boolean accept(DynamicBytes partialBody) {
+            return partialBody.length() <= length;
+        }
+    }
+
+    public boolean accept(Map<String, Object> headers);
+
+    public boolean accept(DynamicBytes partialBody);
+}

data/src/org/httpkit/client/IRespListener.java

@@ -0,0 +1,30 @@
+package org.httpkit.client;
+
+import org.httpkit.HttpStatus;
+import org.httpkit.HttpVersion;
+
+import java.util.Map;
+
+/**
+ * Interface for response received from server
+ * <p/>
+ * A low level interface, can be used for very large file download
+ *
+ * @author feng
+ */
+public interface IRespListener {
+
+    public void onBodyReceived(byte[] buf, int length) throws AbortException;
+
+    public void onCompleted();
+
+    public void onHeadersReceived(Map<String, Object> headers) throws AbortException;
+
+    public void onInitialLineReceived(HttpVersion version, HttpStatus status)
+            throws AbortException;
+
+    /**
+     * protocol error
+     */
+    public void onThrowable(Throwable t);
+}

data/src/org/httpkit/client/IResponseHandler.java

@@ -0,0 +1,18 @@
+package org.httpkit.client;
+
+import java.util.Map;
+
+public interface IResponseHandler {
+
+    /**
+     * get called when all the response are fully received from server
+     *
+     * @param status  HTTP status code, like 200
+     * @param headers Response HTTP headers
+     * @param body    Response body, for text is a String, for binary is a
+     *                InputStream
+     */
+    void onSuccess(int status, Map<String, Object> headers, Object body);
+
+    void onThrowable(Throwable t);
+}

data/src/org/httpkit/client/PersistentConn.java

@@ -0,0 +1,33 @@
+package org.httpkit.client;
+
+import java.net.InetSocketAddress;
+import java.nio.channels.SelectionKey;
+
+public class PersistentConn implements Comparable<PersistentConn> {
+    private final long timeoutTs;
+    public final InetSocketAddress addr;
+    public final SelectionKey key;
+
+    public PersistentConn(long timeoutTs, InetSocketAddress addr, SelectionKey key) {
+        this.timeoutTs = timeoutTs;
+        this.addr = addr;
+        this.key = key;
+    }
+
+    public boolean equals(Object obj) {
+        // for PriorityQueue to remove by key and by addr
+        return addr.equals(obj) || key.equals(obj);
+    }
+
+    public int compareTo(PersistentConn o) {
+        return (int) (timeoutTs - o.timeoutTs);
+    }
+
+    public boolean isTimeout(long now) {
+        return timeoutTs < now;
+    }
+
+    public String toString() {
+        return addr + "; timeout=" + (timeoutTs - System.currentTimeMillis()) + "ms";
+    }
+}