em-http-request 0.2.0

data/.gitignore

@@ -0,0 +1,4 @@
+*.bundle
+*.o
+Makefile
+mkmf.log

data/LICENSE

@@ -0,0 +1,58 @@
+Ruby is copyrighted free software by Yukihiro Matsumoto <matz@netlab.co.jp>.
+You can redistribute it and/or modify it under either the terms of the GPL
+(see COPYING.txt file), or the conditions below:
+
+  1. You may make and give away verbatim copies of the source form of the
+     software without restriction, provided that you duplicate all of the
+     original copyright notices and associated disclaimers.
+
+  2. You may modify your copy of the software in any way, provided that
+     you do at least ONE of the following:
+
+       a) place your modifications in the Public Domain or otherwise
+          make them Freely Available, such as by posting said
+	  modifications to Usenet or an equivalent medium, or by allowing
+	  the author to include your modifications in the software.
+
+       b) use the modified software only within your corporation or
+          organization.
+
+       c) rename any non-standard executables so the names do not conflict
+	  with standard executables, which must also be provided.
+
+       d) make other distribution arrangements with the author.
+
+  3. You may distribute the software in object code or executable
+     form, provided that you do at least ONE of the following:
+
+       a) distribute the executables and library files of the software,
+	  together with instructions (in the manual page or equivalent)
+	  on where to get the original distribution.
+
+       b) accompany the distribution with the machine-readable source of
+	  the software.
+
+       c) give non-standard executables non-standard names, with
+          instructions on where to get the original software distribution.
+
+       d) make other distribution arrangements with the author.
+
+  4. You may modify and include the part of the software into any other
+     software (possibly commercial).  But some files in the distribution
+     are not written by the author, so that they are not under this terms.
+
+     They are gc.c(partly), utils.c(partly), regex.[ch], st.[ch] and some
+     files under the ./missing directory.  See each file for the copying
+     condition.
+
+  5. The scripts and library files supplied as input to or produced as 
+     output from the software do not automatically fall under the
+     copyright of the software, but belong to whomever generated them, 
+     and may be sold commercially, and may be aggregated with this
+     software.
+
+  6. THIS SOFTWARE IS PROVIDED "AS IS" AND WITHOUT ANY EXPRESS OR
+     IMPLIED WARRANTIES, INCLUDING, WITHOUT LIMITATION, THE IMPLIED
+     WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+     PURPOSE.
+

data/README.rdoc

@@ -0,0 +1,71 @@
+= EM-HTTP-Client
+
+EventMachine based HTTP Request interface. Supports streaming response processing, uses Ragel HTTP parser. 
+- Simple interface for single & parallel requests via deferred callbacks
+- Automatic gzip & deflate decoding
+- Basic-Auth support
+- Custom timeouts 
+
+Screencast / Demo of using EM-HTTP-Request: 
+- http://everburning.com/news/eventmachine-screencast-em-http-request/
+
+== Simple client example
+
+  EventMachine.run {
+    http = EventMachine::HttpRequest.new('http://127.0.0.1/').get :query => {'keyname' => 'value'}, :timeout => 10
+ 
+    http.callback {
+      p http.response_header.status
+      p http.response_header
+      p http.response
+    
+      EventMachine.stop
+    }
+  }
+
+== Multi request example
+
+  EventMachine.run {
+    multi = EventMachine::MultiRequest.new
+      
+    # add multiple requests to the multi-handler
+    multi.add(EventMachine::HttpRequest.new('http://www.google.com/').get)
+    multi.add(EventMachine::HttpRequest.new('http://www.yahoo.com/').get)
+      
+    multi.callback  {
+      p multi.responses[:succeeded]
+      p multi.responses[:failed]
+       
+      EventMachine.stop
+    }
+  }
+
+== Basic-Auth example
+
+  EventMachine.run {
+    http = EventMachine::HttpRequest.new('http://www.website.com/').get :head => {'authorization' => ['user', 'pass']}
+ 
+    http.errback { failed }
+    http.callback {
+      p http.response_header
+
+      EventMachine.stop
+    }
+  }
+
+== POST example
+
+  EventMachine.run {
+    http1 = EventMachine::HttpRequest.new('http://www.website.com/').post :body => {"key1" => 1, "key2" => [2,3]}
+    http2 = EventMachine::HttpRequest.new('http://www.website.com/').post :body => "some data"
+      
+    # ...
+  }
+
+== Streaming body processing
+  EventMachine.run {
+    http = EventMachine::HttpRequest.new('http://www.website.com/').get
+    http.stream { |chunk| print chunk }
+
+    # ...
+  }

data/Rakefile

@@ -0,0 +1,103 @@
+require 'rake'
+require 'rake/clean'
+require 'rake/rdoctask'
+require 'rake/gempackagetask'
+require 'fileutils'
+include FileUtils
+
+# copied from EventMachine.
+MAKE = ENV['MAKE'] || if RUBY_PLATFORM =~ /mswin/ # mingw uses make.
+  'nmake'
+else
+  'make'
+end
+
+# Default Rake task is compile
+task :default => :compile
+
+# RDoc
+Rake::RDocTask.new(:rdoc) do |task|
+  task.rdoc_dir = 'doc'
+  task.title    = 'EventMachine::HttpRequest'
+  task.options = %w(--title HttpRequest --main README --line-numbers)
+  task.rdoc_files.include(['lib/**/*.rb'])
+  task.rdoc_files.include(['README', 'LICENSE'])
+end
+
+# Rebuild parser Ragel
+task :ragel do
+  Dir.chdir "ext/http11_client" do
+    target = "http11_parser.c"
+    File.unlink target if File.exist? target
+    sh "ragel http11_parser.rl | rlgen-cd -G2 -o #{target}"
+    raise "Failed to build C source" unless File.exist? target
+  end
+end
+
+task :spec do
+	sh 'spec test/test_*.rb'
+end
+
+def make(makedir)
+  Dir.chdir(makedir) { sh MAKE }
+end
+
+def extconf(dir)
+  Dir.chdir(dir) { ruby "extconf.rb" }
+end
+
+def setup_extension(dir, extension)
+  ext = "ext/#{dir}"
+  ext_so = "#{ext}/#{extension}.#{Config::CONFIG['DLEXT']}"
+  ext_files = FileList[
+    "#{ext}/*.c",
+    "#{ext}/*.h",
+    "#{ext}/extconf.rb",
+    "#{ext}/Makefile",
+    "lib"
+  ] 
+  
+  task "lib" do
+    directory "lib"
+  end
+
+  desc "Builds just the #{extension} extension"
+
+  mf = (extension + '_makefile').to_sym
+
+  task mf do |t|
+    extconf "#{ext}"
+  end
+
+  task extension.to_sym => [mf] do
+    make "#{ext}"
+    cp ext_so, "lib"
+  end
+end
+
+setup_extension("buffer", "em_buffer")
+setup_extension("http11_client", "http11_client")
+
+task :compile => [:em_buffer, :http11_client]
+
+CLEAN.include ['build/*', '**/*.o', '**/*.so', '**/*.a', '**/*.log', 'pkg']
+CLEAN.include ['ext/buffer/Makefile', 'lib/em_buffer.*', 'lib/http11_client.*']
+
+begin
+  require 'jeweler'
+  Jeweler::Tasks.new do |gemspec|
+    gemspec.name = "em-http-request"
+    gemspec.summary = "EventMachine based, async HTTP Request interface"
+    gemspec.description = gemspec.summary
+    gemspec.email = "ilya@igvita.com"
+    gemspec.homepage = "http://github.com/igrigorik/em-http-request"
+    gemspec.authors = ["Ilya Grigorik"]
+    gemspec.add_dependency('eventmachine', '>= 0.12.9')
+    gemspec.add_dependency('addressable', '>= 2.0.0')
+    gemspec.rubyforge_project = "em-http-request"
+  end
+  
+  Jeweler::GemcutterTasks.new
+rescue LoadError
+  puts "Jeweler not available. Install it with: sudo gem install technicalpickles-jeweler -s http://gems.github.com"
+end

data/VERSION

@@ -0,0 +1 @@
+0.2.0

data/examples/fetch.rb

@@ -0,0 +1,30 @@
+require 'rubygems'
+require 'eventmachine'
+require '../lib/em-http'
+
+urls = ARGV
+if urls.size < 1
+  puts "Usage: #{$0} <url> <url> <...>"
+  exit
+end
+
+pending = urls.size
+
+EM.run do
+  urls.each do |url|
+    http = EM::HttpRequest.new(url).get
+    http.callback {
+      puts "#{url}\n#{http.response_header.status} - #{http.response.length} bytes\n"
+      puts http.response
+
+      pending -= 1
+      EM.stop if pending < 1
+    }
+    http.errback {
+      puts "#{url}\n" + http.errors.join("\n")
+
+      pending -= 1
+      EM.stop if pending < 1
+    }
+  end
+end

data/examples/fibered-http.rb

@@ -0,0 +1,39 @@
+require 'eventmachine'
+require 'em-http'
+require 'fiber'
+
+# Using Fibers in Ruby 1.9 to simulate blocking IO / IO scheduling
+# while using the async EventMachine API's
+
+def async_fetch(url)
+  f = Fiber.current
+  http = EventMachine::HttpRequest.new(url).get :timeout => 10
+
+  http.callback { f.resume(http) }
+  http.errback  { f.resume(http) }
+
+  return Fiber.yield
+end
+
+EventMachine.run do
+  Fiber.new{
+
+    puts "Setting up HTTP request #1"
+    data = async_fetch('http://www.google.com/')
+    puts "Fetched page #1: #{data.response_header.status}"
+
+    puts "Setting up HTTP request #2"
+    data = async_fetch('http://www.yahoo.com/')
+    puts "Fetched page #2: #{data.response_header.status}"
+
+    EventMachine.stop
+  }.resume
+end
+
+puts "Done"
+
+#  Setting up HTTP request #1
+#  Fetched page #1: 302
+#  Setting up HTTP request #2
+#  Fetched page #2: 200
+#  Done

data/ext/buffer/em_buffer.c

@@ -0,0 +1,630 @@
+/*
+ * Copyright (C) 2007 Tony Arcieri
+ * You may redistribute this under the terms of the Ruby license.
+ * See LICENSE for details
+ */
+
+#include "ruby.h"
+#include "rubyio.h"
+
+#include <assert.h>
+
+#include <string.h>
+#include <time.h>
+#include <errno.h>
+
+#ifndef GetReadFile
+#define FPTR_TO_FD(fptr) (fptr->fd)
+#else
+#define FPTR_TO_FD(fptr) (fileno(GetReadFile(fptr)))
+#endif
+
+/* Default number of bytes in each node's buffer */
+#define DEFAULT_NODE_SIZE 16384
+
+/* Maximum age of a buffer node in a memory pool, in seconds */
+#define MAX_AGE 60
+
+/* How often to scan the pool for old nodes */
+#define PURGE_INTERVAL 10
+
+struct buffer {
+	time_t last_purged_at;
+  unsigned size, node_size;
+  struct buffer_node *head, *tail;
+  struct buffer_node *pool_head, *pool_tail;
+	
+};
+
+struct buffer_node {
+	time_t last_used_at;
+  unsigned start, end;
+  struct buffer_node *next;
+  unsigned char data[0];
+};
+
+static VALUE mEm = Qnil;
+static VALUE cEm_Buffer = Qnil;
+
+static VALUE Em_Buffer_allocate(VALUE klass);
+static void Em_Buffer_mark(struct buffer *);
+static void Em_Buffer_free(struct buffer *);
+
+static VALUE Em_Buffer_initialize(int argc, VALUE *argv, VALUE self);
+static VALUE Em_Buffer_clear(VALUE self);
+static VALUE Em_Buffer_size(VALUE self);
+static VALUE Em_Buffer_empty(VALUE self);
+static VALUE Em_Buffer_append(VALUE self, VALUE data);
+static VALUE Em_Buffer_prepend(VALUE self, VALUE data);
+static VALUE Em_Buffer_read(int argc, VALUE *argv, VALUE self);
+static VALUE Em_Buffer_to_str(VALUE self);
+static VALUE Em_Buffer_read_from(VALUE self, VALUE io);
+static VALUE Em_Buffer_write_to(VALUE self, VALUE io);
+
+static struct buffer *buffer_new(void);
+static void buffer_clear(struct buffer *buf);
+static void buffer_free(struct buffer *buf);
+static void buffer_gc(struct buffer *buf);
+static void buffer_prepend(struct buffer *buf, char *str, unsigned len);
+static void buffer_append(struct buffer *buf, char *str, unsigned len);
+static void buffer_read(struct buffer *buf, char *str, unsigned len);
+static void buffer_copy(struct buffer *buf, char *str, unsigned len);
+static int buffer_read_from(struct buffer *buf, int fd);
+static int buffer_write_to(struct buffer *buf, int fd);
+
+/* 
+ * High speed buffering geared towards non-blocking I/O.
+ *
+ * Data is stored in a byte queue implemented as a linked list of equal size
+ * chunks.  Since every node in the list is the same size they are easily
+ * memory pooled.  Routines are provided for high speed non-blocking reads
+ * and writes from Ruby IO objects.
+ */
+void Init_em_buffer()
+{
+  mEm = rb_define_module("EventMachine");
+  cEm_Buffer = rb_define_class_under(mEm, "Buffer", rb_cObject);
+  rb_define_alloc_func(cEm_Buffer, Em_Buffer_allocate);
+
+  rb_define_method(cEm_Buffer, "initialize", Em_Buffer_initialize, -1);
+  rb_define_method(cEm_Buffer, "clear", Em_Buffer_clear, 0);
+  rb_define_method(cEm_Buffer, "size", Em_Buffer_size, 0);
+  rb_define_method(cEm_Buffer, "empty?", Em_Buffer_empty, 0);
+  rb_define_method(cEm_Buffer, "<<", Em_Buffer_append, 1);
+  rb_define_method(cEm_Buffer, "append", Em_Buffer_append, 1);
+  rb_define_method(cEm_Buffer, "prepend", Em_Buffer_prepend, 1);
+  rb_define_method(cEm_Buffer, "read", Em_Buffer_read, -1);
+	rb_define_method(cEm_Buffer, "to_str", Em_Buffer_to_str, 0);
+	rb_define_method(cEm_Buffer, "read_from", Em_Buffer_read_from, 1);
+  rb_define_method(cEm_Buffer, "write_to", Em_Buffer_write_to, 1);
+}
+
+static VALUE Em_Buffer_allocate(VALUE klass)
+{
+  return Data_Wrap_Struct(klass, Em_Buffer_mark, Em_Buffer_free, buffer_new());
+}
+
+static void Em_Buffer_mark(struct buffer *buf)
+{
+  /* Walks the pool of unused chunks and frees any that are beyond a certain age */
+  buffer_gc(buf);
+}
+
+static void Em_Buffer_free(struct buffer *buf)
+{
+  buffer_free(buf);
+}
+
+/**
+ *  call-seq:
+ *    EventMachine::Buffer.new(size = DEFAULT_NODE_SIZE) -> EventMachine::Buffer
+ * 
+ * Create a new EventMachine::Buffer with linked segments of the given size
+ */
+static VALUE Em_Buffer_initialize(int argc, VALUE *argv, VALUE self)
+{
+  VALUE node_size_obj;
+  int node_size;
+  struct buffer *buf;
+
+  if(rb_scan_args(argc, argv, "01", &node_size_obj) == 1) {
+    node_size = NUM2INT(node_size_obj);
+
+    if(node_size < 1) rb_raise(rb_eArgError, "invalid buffer size");
+
+    Data_Get_Struct(self, struct buffer, buf);
+
+    /* Make sure we're not changing the buffer size after data has been allocated */
+    assert(!buf->head);
+    assert(!buf->pool_head);
+
+    buf->node_size = node_size;
+  }
+
+  return Qnil;
+}
+
+/**
+ *  call-seq:
+ *    EventMachine::Buffer#clear -> nil
+ * 
+ * Clear all data from the EventMachine::Buffer
+ */
+static VALUE Em_Buffer_clear(VALUE self)
+{
+  struct buffer *buf;
+  Data_Get_Struct(self, struct buffer, buf);
+
+  buffer_clear(buf);
+
+  return Qnil;
+}
+
+/**
+ *  call-seq:
+ *    EventMachine::Buffer#size -> Integer
+ * 
+ * Return the size of the buffer in bytes
+ */
+static VALUE Em_Buffer_size(VALUE self) 
+{
+  struct buffer *buf;
+  Data_Get_Struct(self, struct buffer, buf);
+
+  return INT2NUM(buf->size);
+}
+
+/**
+ *  call-seq:
+ *    EventMachine::Buffer#empty? -> Boolean
+ * 
+ * Is the buffer empty?
+ */
+static VALUE Em_Buffer_empty(VALUE self) 
+{
+  struct buffer *buf;
+  Data_Get_Struct(self, struct buffer, buf);
+
+  return buf->size > 0 ? Qfalse : Qtrue;	
+}
+
+/**
+ *  call-seq:
+ *    EventMachine::Buffer#append(data) -> String
+ * 
+ * Append the given data to the end of the buffer
+ */
+static VALUE Em_Buffer_append(VALUE self, VALUE data)
+{
+  struct buffer *buf;
+  Data_Get_Struct(self, struct buffer, buf);
+
+  /* Is this needed?  Never seen anyone else do it... */
+  data = rb_convert_type(data, T_STRING, "String", "to_str");
+  buffer_append(buf, RSTRING_PTR(data), RSTRING_LEN(data));
+
+  return data;
+}
+
+/**
+ *  call-seq:
+ *    EventMachine::Buffer#prepend(data) -> String
+ * 
+ * Prepend the given data to the beginning of the buffer
+ */
+static VALUE Em_Buffer_prepend(VALUE self, VALUE data)
+{
+  struct buffer *buf;
+  Data_Get_Struct(self, struct buffer, buf);
+
+  data = rb_convert_type(data, T_STRING, "String", "to_str");
+  buffer_prepend(buf, RSTRING_PTR(data), RSTRING_LEN(data));
+
+  return data;
+}
+
+/**
+ *  call-seq:
+ *    EventMachine::Buffer#read(length = nil) -> String
+ * 
+ * Read the specified abount of data from the buffer.  If no value
+ * is given the entire contents of the buffer are returned.  Any data
+ * read from the buffer is cleared.
+ */
+static VALUE Em_Buffer_read(int argc, VALUE *argv, VALUE self)
+{
+  VALUE length_obj, str;
+  int length;
+  struct buffer *buf;
+
+  Data_Get_Struct(self, struct buffer, buf);
+
+  if(rb_scan_args(argc, argv, "01", &length_obj) == 1) {
+    length = NUM2INT(length_obj);
+  } else {
+    if(buf->size == 0)
+      return rb_str_new2("");
+
+    length = buf->size;
+  }
+
+  if(length > buf->size)
+    length = buf->size;
+
+  if(length < 1)
+    rb_raise(rb_eArgError, "length must be greater than zero");
+
+  str = rb_str_new(0, length);
+  buffer_read(buf, RSTRING_PTR(str), length);
+
+  return str;
+}
+
+/**
+ *  call-seq:
+ *    EventMachine::Buffer#to_str -> String
+ * 
+ * Convert the Buffer to a String.  The original buffer is unmodified.
+ */
+static VALUE Em_Buffer_to_str(VALUE self) {
+	VALUE str;
+	struct buffer *buf;
+	
+	Data_Get_Struct(self, struct buffer, buf);
+	
+	str = rb_str_new(0, buf->size);
+	buffer_copy(buf, RSTRING_PTR(str), buf->size);
+  
+  return str;
+}
+
+/**
+ *  call-seq:
+ *    EventMachine::Buffer#read_from(io) -> Integer
+ * 
+ * Perform a nonblocking read of the the given IO object and fill
+ * the buffer with any data received.  The call will read as much
+ * data as it can until the read would block.
+ */
+static VALUE Em_Buffer_read_from(VALUE self, VALUE io) {
+	struct buffer *buf;
+#if HAVE_RB_IO_T
+  rb_io_t *fptr;
+#else
+  OpenFile *fptr;
+#endif
+
+  Data_Get_Struct(self, struct buffer, buf);
+  GetOpenFile(rb_convert_type(io, T_FILE, "IO", "to_io"), fptr);
+  rb_io_set_nonblock(fptr);
+
+  return INT2NUM(buffer_read_from(buf, FPTR_TO_FD(fptr)));
+}
+
+/**
+ *  call-seq:
+ *    EventMachine::Buffer#write_to(io) -> Integer
+ * 
+ * Perform a nonblocking write of the buffer to the given IO object.
+ * As much data as possible is written until the call would block.
+ * Any data which is written is removed from the buffer.
+ */
+static VALUE Em_Buffer_write_to(VALUE self, VALUE io) {
+  struct buffer *buf;
+#if HAVE_RB_IO_T 
+  rb_io_t *fptr;
+#else
+  OpenFile *fptr;
+#endif
+
+  Data_Get_Struct(self, struct buffer, buf);
+  GetOpenFile(rb_convert_type(io, T_FILE, "IO", "to_io"), fptr);
+  rb_io_set_nonblock(fptr);
+
+  return INT2NUM(buffer_write_to(buf, FPTR_TO_FD(fptr)));
+}
+
+/*
+ * Ruby bindings end here.  Below is the actual implementation of 
+ * the underlying data structures.
+ */
+
+/* Create a new buffer */
+static struct buffer *buffer_new(void)
+{
+  struct buffer *buf;
+
+  buf = (struct buffer *)xmalloc(sizeof(struct buffer));
+  buf->head = buf->tail = buf->pool_head = buf->pool_tail = 0;
+  buf->size = 0;
+  buf->node_size = DEFAULT_NODE_SIZE;
+	time(&buf->last_purged_at);
+	
+  return buf;
+}
+
+/* Clear all data from a buffer */
+static void buffer_clear(struct buffer *buf)
+{
+  struct buffer_node *tmp;
+
+  /* Move everything into the buffer pool */
+  if(!buf->pool_tail)
+    buf->pool_head = buf->pool_tail = buf->head;
+  else
+    buf->pool_tail->next = buf->head;
+
+  buf->head = buf->tail = 0;
+  buf->size = 0;
+}
+
+/* Free a buffer */
+static void buffer_free(struct buffer *buf) 
+{
+  struct buffer_node *tmp;
+
+  buffer_clear(buf);
+
+  while(buf->pool_head) {
+    tmp = buf->pool_head;
+    buf->pool_head = tmp->next;
+    free(tmp);
+  }
+
+  free(buf);
+}
+
+/* Run through the pool and find elements that haven't been used for awhile */
+static void buffer_gc(struct buffer *buf)
+{
+  struct buffer_node *cur, *tmp;
+  time_t now;
+  time(&now);
+
+	/* Only purge if we've passed the purge interval */
+	if(now - buf->last_purged_at < PURGE_INTERVAL)
+		return;
+		
+	buf->last_purged_at = now;
+
+  while(buf->pool_head && now - buf->pool_head->last_used_at >= MAX_AGE) {
+    tmp = buf->pool_head;
+    buf->pool_head = buf->pool_head->next;
+    free(tmp);
+  }
+
+  if(!buf->pool_head)
+		buf->pool_tail = 0;
+}
+
+/* Create a new buffer_node (or pull one from the memory pool) */
+static struct buffer_node *buffer_node_new(struct buffer *buf)
+{
+  struct buffer_node *node;
+
+  /* Pull from the memory pool if available */
+  if(buf->pool_head) {
+    node = buf->pool_head;
+    buf->pool_head = node->next;
+
+    if(node->next)
+      node->next = 0;
+    else
+      buf->pool_tail = 0;
+  } else {
+    node = (struct buffer_node *)xmalloc(sizeof(struct buffer_node) + buf->node_size);
+    node->next = 0;
+  }
+
+  node->start = node->end = 0;
+  return node;
+}
+
+/* Free a buffer node (i.e. return it to the memory pool) */
+static void buffer_node_free(struct buffer *buf, struct buffer_node *node)
+{
+  /* Store when the node was freed */
+  time(&node->last_used_at);
+
+  node->next = buf->pool_head;
+  buf->pool_head = node;
+
+  if(!buf->pool_tail)
+    buf->pool_tail = node;
+}
+
+/* Prepend data to the front of the buffer */
+static void buffer_prepend(struct buffer *buf, char *str, unsigned len)
+{
+  struct buffer_node *node, *tmp;
+  buf->size += len;
+
+  /* If it fits in the beginning of the head */
+  if(buf->head && buf->head->start >= len) {
+    buf->head->start -= len;
+    memcpy(buf->head->data + buf->head->start, str, len);
+  } else {
+    node = buffer_node_new(buf);
+    node->next = buf->head;
+    buf->head = node;
+    if(!buf->tail) buf->tail = node;
+
+    while(len > buf->node_size) {
+      memcpy(node->data, str, buf->node_size);
+      node->end = buf->node_size;
+
+      tmp = buffer_node_new(buf);
+      tmp->next = node->next;
+      node->next = tmp;
+
+      if(buf->tail == node) buf->tail = tmp;
+      node = tmp;
+
+      str += buf->node_size;
+      len -= buf->node_size;
+    }
+
+    if(len > 0) {
+      memcpy(node->data, str, len);
+      node->end = len;
+    }
+  }
+}
+
+/* Append data to the front of the buffer */
+static void buffer_append(struct buffer *buf, char *str, unsigned len)
+{
+  unsigned nbytes;
+  buf->size += len;
+
+  /* If it fits in the remaining space in the tail */
+  if(buf->tail && len <= buf->node_size - buf->tail->end) {
+    memcpy(buf->tail->data + buf->tail->end, str, len);
+    buf->tail->end += len;
+    return;
+  }
+
+  /* Empty list needs initialized */
+  if(!buf->head) {
+    buf->head = buffer_node_new(buf);
+    buf->tail = buf->head;
+  }
+
+  /* Build links out of the data */
+  while(len > 0) {
+    nbytes = buf->node_size - buf->tail->end;
+    if(len < nbytes) nbytes = len;
+    
+    memcpy(buf->tail->data + buf->tail->end, str, nbytes);
+    str += nbytes;    
+    len -= nbytes;
+    
+    buf->tail->end += nbytes;
+
+    if(len > 0) {
+      buf->tail->next = buffer_node_new(buf);
+      buf->tail = buf->tail->next;
+    }
+  }
+}
+
+/* Read data from the buffer (and clear what we've read) */
+static void buffer_read(struct buffer *buf, char *str, unsigned len)
+{
+  unsigned nbytes;
+  struct buffer_node *tmp;
+
+  while(buf->size > 0 && len > 0) {
+    nbytes = buf->head->end - buf->head->start;
+    if(len < nbytes) nbytes = len;
+
+    memcpy(str, buf->head->data + buf->head->start, nbytes);
+    str += nbytes;
+    len -= nbytes;
+
+    buf->head->start += nbytes;
+    buf->size -= nbytes;
+
+    if(buf->head->start == buf->head->end) {
+      tmp = buf->head;
+      buf->head = tmp->next;
+      buffer_node_free(buf, tmp);
+
+      if(!buf->head) buf->tail = 0;
+    }
+  }
+}
+
+/* Copy data from the buffer without clearing it */
+static void buffer_copy(struct buffer *buf, char *str, unsigned len)
+{
+  unsigned nbytes;
+  struct buffer_node *node;
+
+	node = buf->head;
+  while(node && len > 0) {
+    nbytes = node->end - node->start;
+    if(len < nbytes) nbytes = len;
+
+    memcpy(str, node->data + node->start, nbytes);
+    str += nbytes;
+    len -= nbytes;
+
+    if(node->start + nbytes == node->end)
+			node = node->next;
+  }
+}
+
+/* Write data from the buffer to a file descriptor */
+static int buffer_write_to(struct buffer *buf, int fd)
+{
+  int bytes_written, total_bytes_written = 0;
+  struct buffer_node *tmp;
+
+  while(buf->head) {
+    bytes_written = write(fd, buf->head->data + buf->head->start, buf->head->end - buf->head->start);
+
+    /* If the write failed... */
+    if(bytes_written < 0) {
+      if(errno != EAGAIN)
+        rb_sys_fail("write");
+
+      return total_bytes_written;
+    }
+
+    total_bytes_written += bytes_written;
+    buf->size -= bytes_written;
+
+    /* If the write blocked... */
+    if(bytes_written < buf->head->end - buf->head->start) {
+      buf->head->start += bytes_written;
+      return total_bytes_written;
+    }
+
+    /* Otherwise we wrote the whole buffer */
+    tmp = buf->head;
+    buf->head = tmp->next;
+    buffer_node_free(buf, tmp);
+
+    if(!buf->head) buf->tail = 0;
+  }
+
+  return total_bytes_written;
+}
+
+/* Read data from a file descriptor to a buffer */
+/* Append data to the front of the buffer */
+static int buffer_read_from(struct buffer *buf, int fd)
+{
+	int bytes_read, total_bytes_read = 0;
+  unsigned nbytes;
+
+  /* Empty list needs initialized */
+  if(!buf->head) {
+    buf->head = buffer_node_new(buf);
+    buf->tail = buf->head;
+  }
+
+	do {
+	  nbytes = buf->node_size - buf->tail->end;
+		bytes_read = read(fd, buf->tail->data + buf->tail->end, nbytes);
+	
+		if(bytes_read < 1) {
+			if(errno != EAGAIN)
+        rb_sys_fail("read");
+			
+			return total_bytes_read;
+		}
+		
+		total_bytes_read += bytes_read; 
+		buf->tail->end += nbytes;
+		buf->size += nbytes;
+		
+		if(buf->tail->end == buf->node_size) {
+      buf->tail->next = buffer_node_new(buf);
+      buf->tail = buf->tail->next;
+		}
+	} while(bytes_read == nbytes);
+	
+	return total_bytes_read;
+}