mongrel_esi 0.5.1 → 0.5.2

data/ext/esi/parser.h

@@ -7,7 +7,8 @@
 #include <sys/types.h>
 
 /* how much output to hold in memory before sending out */
-#define ESI_OUTPUT_BUFFER_SIZE 1024
+#define ESI_OUTPUT_BUFFER_SIZE 4096
+#define ESI_ECHOBUFFER_SIZE 128
 
 char *esi_strndup( const char *str, size_t len );
 
@@ -28,8 +29,10 @@ char *esi_strndup( const char *str, size_t len );
  *
  * */
 typedef struct _ESIAttr {
-  char *name;
-  char *value;
+  const char *name;
+  const char *value;
+  size_t name_length;
+  size_t value_length;
   struct _ESIAttr *next;
 }ESIAttribute;
 
@@ -59,8 +62,10 @@ typedef struct _ESIParser {
   void *user_data;
 
   const char *mark;
-  size_t overflow_data_size; /* current size of the overflow data buffer */
+  size_t overflow_data_size; /* amount of the overflow buffer being used */
+  size_t overflow_data_allocated; /* amount of memory allocated to use for the overflow buffer */
   char *overflow_data; /* overflow buffer if execute finishes and we are not in a final state store the parse data */
+  unsigned using_overflow:1; /* if this is 1, overflow data is in use. */
 
   size_t echobuffer_allocated; /* amount of memory allocated for the echobuffer */
   size_t echobuffer_index; /* current write position of the last echo'ed character */

data/ext/esi/parser.rl

@@ -64,6 +64,21 @@ static void esi_parser_echo_char( ESIParser *parser, char ch )
     esi_parser_flush_output( parser );
   }
 }
+static void esi_parser_send_buffer( ESIParser *parser, const char *seq, size_t len )
+{
+  const char *nseq = seq + len;
+  size_t nlen = parser->output_buffer_size + len;
+
+  if( nlen < ESI_OUTPUT_BUFFER_SIZE ) {
+    memcpy( parser->output_buffer + parser->output_buffer_size, seq, len ); 
+    parser->output_buffer_size += len;
+  }
+  else if( nlen > ESI_OUTPUT_BUFFER_SIZE ) {
+    while( seq != nseq ) {
+      esi_parser_echo_char( parser, *seq++ );
+    }
+  }
+}
 /* send any buffered characters to the output handler. 
  * This happens when we enter a case such as <em>  where the
  * first two characters < and e  match the <esi:  expression
@@ -99,6 +114,7 @@ static void esi_parser_concat_to_echobuffer( ESIParser *parser, char ch )
     /* double the echobuffer size 
      * we're getting some crazy input if this case ever happens
      */
+    printf( "increase echobuffer: %d, %s\n", parser->echobuffer_allocated, parser->echobuffer );
     parser->echobuffer_allocated *= 2;
     parser->echobuffer = (char*)realloc( parser->echobuffer, parser->echobuffer_allocated );
   }
@@ -165,9 +181,8 @@ static void rtrim_pointer( const char **ptr, const char *bounds, size_t *len )
     /* send the start tag and end tag message */
     esi_parser_flush_output( parser );
     parser->start_tag_handler( data, parser->tag_text, parser->tag_text_length, parser->attributes, parser->user_data );
-    esi_parser_flush_output( parser );
     parser->end_tag_handler( data, parser->tag_text, parser->tag_text_length, parser->user_data );
-    esi_parser_flush_output( parser );
+    //printf("\t[see inline tag]\n");
 
     /* mark the position */
     parser->tag_text = NULL;
@@ -187,7 +202,7 @@ static void rtrim_pointer( const char **ptr, const char *bounds, size_t *len )
     /* send the start and end tag message */
     esi_parser_flush_output( parser );
     parser->start_tag_handler( data, parser->tag_text, parser->tag_text_length, parser->attributes, parser->user_data );
-    esi_parser_flush_output( parser );
+    //printf("\t[see start tag with attributes]\n");
 
     parser->tag_text = NULL;
     parser->tag_text_length = 0;
@@ -253,7 +268,7 @@ static void rtrim_pointer( const char **ptr, const char *bounds, size_t *len )
 
     esi_parser_flush_output( parser );
     parser->start_tag_handler( data, parser->tag_text, parser->tag_text_length, NULL, parser->user_data );
-    esi_parser_flush_output( parser );
+    //printf("\t[see start tag]\n");
 
     esi_parser_echobuffer_clear( parser );
   }
@@ -271,7 +286,7 @@ static void rtrim_pointer( const char **ptr, const char *bounds, size_t *len )
     
     esi_parser_flush_output( parser );
     parser->end_tag_handler( data, parser->tag_text, parser->tag_text_length, parser->user_data );
-    esi_parser_flush_output( parser );
+    //printf("\t[see end tag]\n");
 
     esi_parser_echobuffer_clear( parser );
   }
@@ -319,8 +334,10 @@ char *esi_strndup( const char *str, size_t len )
 ESIAttribute *esi_attribute_new( const char *name, size_t name_length, const char *value, size_t value_length )
 {
   ESIAttribute *attr = (ESIAttribute*)malloc(sizeof(ESIAttribute));
-  attr->name = esi_strndup(name, name_length);
-  attr->value = esi_strndup(value, value_length);
+  attr->name = name;//esi_strndup(name, name_length);
+  attr->value = value;//esi_strndup(value, value_length);
+  attr->name_length = name_length;
+  attr->value_length = value_length;
   attr->next = NULL;
   return attr;
 }
@@ -331,16 +348,16 @@ ESIAttribute *esi_attribute_copy( ESIAttribute *attribute )
   if( !attribute ){ return NULL; }
 
   // copy the first attribute
-  nattr = esi_attribute_new( attribute->name, strlen( attribute->name ),
-                             attribute->value, strlen( attribute->value ) );
+  nattr = esi_attribute_new( attribute->name, attribute->name_length,
+                             attribute->value, attribute->value_length );
   // save a pointer for return
   head = nattr;
   // copy next attributes
   attribute = attribute->next;
   while( attribute ) {
     // set the next attribute
-    nattr->next = esi_attribute_new( attribute->name, strlen( attribute->name ),
-                                     attribute->value, strlen( attribute->value ) );
+    nattr->next = esi_attribute_new( attribute->name, attribute->name_length,
+                                     attribute->value, attribute->value_length );
     // next attribute
     nattr = nattr->next;
     attribute = attribute->next;
@@ -352,8 +369,8 @@ void esi_attribute_free( ESIAttribute *attribute )
 {
   ESIAttribute *ptr;
   while( attribute ){
-    free( attribute->name );
-    free( attribute->value );
+//    free( attribute->name );
+//    free( attribute->value );
     ptr = attribute->next;
     free( attribute );
     attribute = ptr;
@@ -370,9 +387,11 @@ ESIParser *esi_parser_new()
   parser->attr_value = NULL;
   parser->overflow_data_size = 0;
   parser->overflow_data = NULL;
+  parser->overflow_data_allocated = 0;
+  parser->using_overflow = 0;
 
   /* allocate ESI_OUTPUT_BUFFER_SIZE bytes for the echobuffer */
-  parser->echobuffer_allocated = ESI_OUTPUT_BUFFER_SIZE;
+  parser->echobuffer_allocated = ESI_ECHOBUFFER_SIZE;
   parser->echobuffer_index = -1;
   parser->echobuffer = (char*)malloc(sizeof(char)*parser->echobuffer_allocated);
 
@@ -393,6 +412,7 @@ void esi_parser_free( ESIParser *parser )
   if( parser->overflow_data ){ free( parser->overflow_data ); }
 
   free( parser->echobuffer );
+
   esi_attribute_free( parser->attributes );
 
   free( parser );
@@ -465,6 +485,54 @@ esi_parser_scan_for_start( ESIParser *parser, const char *data, size_t length )
   }
 }
 
+static void 
+esi_parser_append_data_to_overflow_data( ESIParser *parser, const char *data, size_t length )
+{
+  int mark_offset       = 0;
+  int tag_text_offset   = 0;
+  int attr_key_offset   = 0;
+  int attr_value_offset = 0;
+
+  if( parser->using_overflow ) {
+    // recompute mark, tag_text, attr_key, and attr_value as they all exist within overflow_data
+    mark_offset       = compute_offset( parser->mark,       parser->overflow_data );
+    tag_text_offset   = compute_offset( parser->tag_text,   parser->overflow_data );
+    attr_key_offset   = compute_offset( parser->attr_key,   parser->overflow_data );
+    attr_value_offset = compute_offset( parser->attr_value, parser->overflow_data );
+  }
+  else {
+    // recompute mark, tag_text, attr_key, and attr_value as they all exist within data
+    mark_offset       = compute_offset( parser->mark,       data );
+    tag_text_offset   = compute_offset( parser->tag_text,   data );
+    attr_key_offset   = compute_offset( parser->attr_key,   data );
+    attr_value_offset = compute_offset( parser->attr_value, data );
+  }
+
+  if( parser->overflow_data ) {
+    // check if we need to resize or grow the buffer
+    if( (parser->overflow_data_size + length) > parser->overflow_data_allocated ) {
+      parser->overflow_data_allocated += length;
+//      printf( "realloc: %d bytes, using %d\n", (int)parser->overflow_data_allocated, (int)(parser->overflow_data_size+length) );
+      parser->overflow_data = (char*)realloc( parser->overflow_data, sizeof(char)*(parser->overflow_data_allocated) );
+    }
+    memcpy( parser->overflow_data+parser->overflow_data_size, data, length );
+    parser->overflow_data_size += length;
+  }
+  else {
+    parser->overflow_data_allocated = ESI_OUTPUT_BUFFER_SIZE > length ? ESI_OUTPUT_BUFFER_SIZE : length;
+    parser->overflow_data = (char*)malloc( sizeof( char ) * parser->overflow_data_allocated );
+    memcpy( parser->overflow_data, data, length );
+    parser->overflow_data_size = length;
+//    printf( "malloc: %d for %d bytes\n", (int)parser->overflow_data_allocated, (int)length );
+  }
+
+  // in our new memory space mark will now be
+  parser->mark = ( mark_offset >= 0 ) ? parser->overflow_data + mark_offset : NULL;
+  parser->tag_text = ( tag_text_offset >= 0 ) ? parser->overflow_data + tag_text_offset : NULL;
+  parser->attr_key = ( attr_key_offset >= 0 ) ? parser->overflow_data + attr_key_offset : NULL;
+  parser->attr_value = ( attr_value_offset >= 0 ) ? parser->overflow_data + attr_value_offset : NULL;
+}
+
 /* accept an arbitrary length string buffer
  * when this methods exits it determines if an end state was reached
  * if no end state was reached it saves the full input into an internal buffer
@@ -482,49 +550,31 @@ int esi_parser_execute( ESIParser *parser, const char *data, size_t length )
   if( length == 0 ){ return cs; }
 
   /* scan data for any '<esi:' start sequences, /<$/, /<e$/, /<es$/, /<esi$/, /<esi:$/ */
-  if( cs == esi_start ) { 
+  if( cs == esi_start || cs == 0 ) { 
     pindex = esi_parser_scan_for_start( parser, data, length );
-    if( pindex == -1 ) {
-      for( pindex = 0; pindex < length; ++pindex ) {
-        esi_parser_echo_char( parser, data[pindex] );
-      }
-      return cs;
+    if( pindex == -1 ) { /* this means there are no start sequences in the buffer we recieved */
+      esi_parser_send_buffer( parser, data, length );
+      return cs; /* no esi start sequences send it all out and break out early */
     }
   }
 
-  /* there's an existing overflow buffer data append the new data to the existing data */
-  if( parser->overflow_data && parser->overflow_data_size > 0 ) {
-
-    // recompute mark, tag_text, attr_key, and attr_value since they all exist within overflow_data
-    int mark_offset = compute_offset( parser->mark, parser->overflow_data );
-    int tag_text_offset = compute_offset( parser->tag_text, parser->overflow_data );
-    int attr_key_offset = compute_offset( parser->attr_key, parser->overflow_data );
-    int attr_value_offset = compute_offset( parser->attr_value, parser->overflow_data );
-
-    parser->overflow_data = (char*)realloc( parser->overflow_data, sizeof(char)*(parser->overflow_data_size+length) );
-    memcpy( parser->overflow_data+parser->overflow_data_size, data, length );
+  /* there's an existing overflow buffer and it's being used, append the new data */
+  //if( parser->overflow_data && parser->overflow_data_size > 0 ) {
+  if( parser->using_overflow ) {
+    int prev_overflow_size = parser->overflow_data_size; // save the current overflow size
+    esi_parser_append_data_to_overflow_data( parser, data, length );
 
-    p = parser->overflow_data + parser->overflow_data_size;
- 
-    // in our new memory space mark will now be
-    parser->mark = ( mark_offset >= 0 ) ? parser->overflow_data + mark_offset : NULL;
-    parser->tag_text = ( tag_text_offset >= 0 ) ? parser->overflow_data + tag_text_offset : NULL;
-    parser->attr_key = ( attr_key_offset >= 0 ) ? parser->overflow_data + attr_key_offset : NULL;
-    parser->attr_value = ( attr_value_offset >= 0 ) ? parser->overflow_data + attr_value_offset : NULL;
- 
+    /* set parser variables */
+    p = parser->overflow_data + prev_overflow_size;
     data = parser->overflow_data;
-    parser->overflow_data_size = length = length + parser->overflow_data_size;
-//    printf( "grow overflow data: %ld\n", parser->overflow_data_size );
+    length = parser->overflow_data_size;
     pe = data + length;
-
   }
 
   if( !parser->mark ) {
     parser->mark = p;
   }
 
-//  printf( "cs: %d, ", cs );
-
   %% write exec;
 
   parser->cs = cs;
@@ -532,37 +582,13 @@ int esi_parser_execute( ESIParser *parser, const char *data, size_t length )
   if( cs != esi_start && cs != 0 ) {
 
     /* reached the end and we're not at a termination point save the buffer as overflow */
-    if( !parser->overflow_data ){
-      // recompute mark, tag_text, attr_key, and attr_value since they all exist within overflow_data
-      int mark_offset = compute_offset( parser->mark, data );
-      int tag_text_offset = compute_offset( parser->tag_text, data );
-      int attr_key_offset = compute_offset( parser->attr_key, data );
-      int attr_value_offset = compute_offset( parser->attr_value, data );
-      //debug_string( "mark before move", parser->mark, 1 );
-
-      if( ESI_OUTPUT_BUFFER_SIZE > length ) {
-        parser->echobuffer_allocated = ESI_OUTPUT_BUFFER_SIZE;
-      }
-      else {
-        parser->echobuffer_allocated = length;
-      }
-      parser->overflow_data = (char*)malloc( sizeof( char ) * parser->echobuffer_allocated );
-      memcpy( parser->overflow_data, data, length );
-      parser->overflow_data_size = length;
-      //printf( "allocate overflow data: %ld\n", parser->echobuffer_allocated );
-
-      // in our new memory space mark will now be
-      parser->mark = ( mark_offset >= 0 ) ? parser->overflow_data + mark_offset : NULL;
-      parser->tag_text = ( tag_text_offset >= 0 ) ? parser->overflow_data + tag_text_offset : NULL;
-      parser->attr_key = ( attr_key_offset >= 0 ) ? parser->overflow_data + attr_key_offset : NULL;
-      parser->attr_value = ( attr_value_offset >= 0 ) ? parser->overflow_data + attr_value_offset : NULL;
-      //if( parser->mark ){ debug_string( "mark after  move", parser->mark, 1 ); } else { printf( "mark is now empty\n" ); }
+    if( !parser->using_overflow ) { 
+      esi_parser_append_data_to_overflow_data( parser, data, length );
+      parser->using_overflow = 1;
     }
 
-  }else if( parser->overflow_data ) {
-    /* dump the overflow buffer execution ended at a final state */
-    free( parser->overflow_data );
-    parser->overflow_data = NULL;
+  } else if ( parser->using_overflow ) { 
+    parser->using_overflow = 0;
     parser->overflow_data_size = 0;
   }
 

data/ext/esi/test/test.c

@@ -38,11 +38,16 @@ TagInfo* tag_info_new( const char *name, size_t len, ESIAttribute *attributes )
 static
 void tag_info_show( TagInfo *ti )
 {
+  char buffer1[1024], buffer2[1024];
   ESIAttribute *attrs = ti->attributes;
   if( attrs ) {
     printf("%s{", ti->name );
     while( attrs ) {
-      printf( "(%s=>%s),", attrs->name, attrs->value );
+      memcpy( buffer1, attrs->name, attrs->name_length );
+      buffer1[attrs->name_length] = '\0';
+      memcpy( buffer2, attrs->value, attrs->value_length );
+      buffer2[attrs->value_length] = '\0';
+      printf( "(%s=>%s),", buffer1, buffer2 );
       attrs = attrs->next;
     }
     printf("}\n");
@@ -198,7 +203,7 @@ static void feed_data( ESIParser *parser, const char *data )
 #define TEST_WITH_FILE(file)\
   int size = 0;\
   FILE *input = NULL;\
-  char buffer[1024]; /* intentially using a small buffer size */\
+  char buffer[4096]; \
   TEST_INIT\
 \
   input = fopen( file, "r" );\
@@ -207,7 +212,7 @@ static void feed_data( ESIParser *parser, const char *data )
     return;\
   }\
 \
-  while( (size = fread( buffer, sizeof(char), 1023, input )) > 0 ){\
+  while( (size = fread( buffer, sizeof(char), 4095, input )) > 0 ){\
     esi_parser_execute( parser, buffer, size );\
   }\
 \
@@ -281,13 +286,13 @@ static void test_small_buffer( ESIParser *parser )
   int i, len = strlen(input);
 
   TEST_INIT
+//  printf("\n");
 
   for( i = 0; i < len; i += input_size ) {
     if( i + input_size <= len ) {
       esi_parser_execute( parser, (input+i), input_size );
     }
     else {
-      printf("using: %d\n", len - i );
       esi_parser_execute( parser, (input+i), len - i );
     }
   }
@@ -545,10 +550,10 @@ static void test_large_chunked_file( ESIParser *parser )
   buffer_end = buffer + size;
 
   for( chunk_start = buffer; chunk_start != buffer_end; ) {
-    while( *chunk_start != '{' && chunk_start != buffer_end ) { ++chunk_start; }
+    while( chunk_start != buffer_end && *chunk_start != '{' ) { ++chunk_start; }
     if( chunk_start == buffer_end ){ break; }
     chunk_end = chunk_start;
-    while( *chunk_end != '}' && chunk_end != buffer_end ) { ++chunk_end; }
+    while( chunk_end != buffer_end && *chunk_end != '}' ) { ++chunk_end; }
     ++chunk_start;
     if( chunk_start > buffer_end ){ break; }
 

data/lib/esi/cache.rb

@@ -92,8 +92,12 @@ module ESI
   #
   class MemcachedCache < Cache
     require 'rubygems'
-    gem 'memcache-client' if respond_to? :gem
-    require 'memcache'
+    begin
+      require 'memcached'
+    rescue => e
+      puts "using memcache client"
+      require 'memcache'
+    end
     include ESI::Log
 
     def initialize( options )
@@ -175,7 +179,7 @@ module ESI
     def get( uri, params )
       key = cache_key(uri,params)
       fragment = @semaphore.synchronize { @cache[key] }
-      fragment.dup if fragment and fragment.respond_to?(:dup)
+      fragment.dup if fragment and fragment.valid? and fragment.respond_to?(:dup)
     end
  
     def put( uri, params, max_age, body )

data/lib/esi/config.rb

@@ -1,6 +1,7 @@
 # Copyright (c) 2008 Todd A. Fisher
 # see LICENSE
 require 'ostruct'
+require 'esi/router'
 
 module ESI
   # This file provides configuration options to mongrel-esi
@@ -108,6 +109,8 @@ module ESI
         yield cache_options
         if cache_options.memcached?
           @config[:cache] = 'memcached'
+        else
+          @config[:cache] = 'ruby'
         end
         @config[:cache_options] = cache_options.options
       else
@@ -135,7 +138,7 @@ module ESI
     end
 
     def router
-      Router.new( @config[:routing] )
+      ESI::Router.new( @config[:routing] )
     end
  
     class ConfigRouter
@@ -169,13 +172,6 @@ module ESI
       @config[:routing] = config_router.routes
     end
 
-    # return true/false depending on the value of 
-    # invalidator: on/off
-    # within in config/esi.yml
-    def start_invalidator?
-      @config[:invalidator]
-    end
-
     def self.define( listeners )
       listeners.each do|host,server|
         esi_handlers = server.classifier.handler_map.select do|uri,handler|

data/lib/esi/dispatcher.rb

@@ -1,22 +1,31 @@
+require 'mongrel'
 require 'esi/proxy'
+require 'esi/logger'
 
 module ESI
 
   class Dispatcher < Mongrel::HttpHandler
-    attr_reader :config, :router
+    attr_reader :config
+    include ESI::Log
 
     Thread.abort_on_exception = false
 
     def initialize( options )
       super()
       @config = ESI::Config.new( options )
-      if @config.start_invalidator?
-        ESI::Invalidator.start( self )
-      end
     end
 
     def process(request, response)
-      ESI::Proxy.new(@config).process(request,response)
+      start = Time.now
+
+      url = @config.router.url_for(request.params["REQUEST_URI"])
+
+      chunk_count, bytes_sent, status, sent_from_cache = ESI::Proxy.new(@config).process(url, request,response)
+
+    rescue => e
+      log_error "\n#{e.message}: error at #{e.backtrace.first} msg at #{__FILE__}:#{__LINE__}\n"
+    ensure
+      log_request "\n#{url}, #{Time.now - start} seconds with status #{status} #{sent_from_cache ? "from cache" : ''} and #{chunk_count} chunks, #{bytes_sent} bytes\n"
     end
 
   end