mongrel_esi 0.4.1 → 0.5.0

data/ext/esi/test/parser.h

@@ -6,6 +6,8 @@
 #define ESI_PARSER_H
 #include <sys/types.h>
 
+/* how much output to hold in memory before sending out */
+#define ESI_OUTPUT_BUFFER_SIZE 1024
 
 char *esi_strndup( const char *str, size_t len );
 
@@ -75,6 +77,11 @@ typedef struct _ESIParser {
 
   ESIAttribute *attributes, *last;
 
+  /* this memory will be pass to the output_cb when either it's full
+   * or eof is encountered */
+  char output_buffer[ESI_OUTPUT_BUFFER_SIZE+1];
+  size_t output_buffer_size;
+
   start_tag_cb start_tag_handler;
   end_tag_cb end_tag_handler;
   output_cb output_handler;

data/ext/esi/test/parser.rl

@@ -1,9 +1,22 @@
+/** 
+ * Copyright (c) 2008 Todd A. Fisher
+ * see LICENSE
+ */
 #include <stdio.h>
 #include <stdlib.h>
 #include <string.h>
 #include <ctype.h>
 #include "parser.h"
 
+#ifdef DEBUG
+static void debug_string( const char *msg, const char *str, size_t len )
+{
+  char *pstr = esi_strndup( str, len );
+  printf( "%s :'%s'\n", msg, pstr );
+  free( pstr );
+}
+#endif
+
 /* define default callbacks */
 static void 
 esi_parser_default_start_cb( const void *data,
@@ -26,58 +39,102 @@ esi_parser_default_output_cp(const void *data,
                              void *user_data)
 {
 }
-#ifdef DEBUG
-static void debug_string( const char *msg, const char *str, size_t len )
+
+/* 
+ * flush output buffer
+ */
+static void esi_parser_flush_output( ESIParser *parser )
 {
-  char *pstr = esi_strndup( str, len );
-  printf( "%s :'%s'\n", msg, pstr );
-  free( pstr );
+  if( parser->output_buffer_size > 0 ) {
+    //debug_string( "esi_parser_flush_output:", parser->output_buffer, parser->output_buffer_size );
+    parser->output_handler( (void*)parser->output_buffer, parser->output_buffer_size, parser->user_data );
+    parser->output_buffer_size = 0;
+  }
 }
-#endif
-
+/* send the character to the output handler marking it 
+ * as ready for consumption, e.g. not an esi tag
+ */
 static void esi_parser_echo_char( ESIParser *parser, char ch )
 {
-  parser->output_handler( (void*)&ch, 1, parser->user_data );
+  parser->output_buffer[parser->output_buffer_size++] = ch;
+  if( parser->output_buffer_size == ESI_OUTPUT_BUFFER_SIZE ) {
+    // flush the buffer to the consumer
+    esi_parser_flush_output( parser );
+  }
 }
-
+/* send any buffered characters to the output handler. 
+ * This happens when we enter a case such as <em>  where the
+ * first two characters < and e  match the <esi:  expression
+ */
 static void esi_parser_echo_buffer( ESIParser *parser )
 {
-//  debug_string( "echobuffer", parser->echobuffer, parser->echobuffer_index+1 );
-  parser->output_handler( parser->echobuffer, parser->echobuffer_index+1, parser->user_data );
+  size_t i = 0, len = parser->echobuffer_index + 1;;
+  //debug_string( "echobuffer", parser->echobuffer, parser->echobuffer_index+1 );
+  //parser->output_handler( parser->echobuffer, parser->echobuffer_index+1, parser->user_data );
+  for( ; i < len; ++i ) {
+    esi_parser_echo_char( parser, parser->echobuffer[i] );
+  }
 }
-
+/*
+ * clear the buffer, no buffered characters should be emitted .
+ * e.g. we matched an esi tag completely and all buffered characters can be tossed out
+ */
 static void esi_parser_echobuffer_clear( ESIParser *parser )
 {
   parser->echobuffer_index = -1;
 }
 
+/*
+ * add a character to the echobuffer. 
+ * this happens when we can't determine if the character is allowed to be sent to the client device
+ * e.g. matching <e  it's not yet determined if these characters are safe to send or not
+ */
 static void esi_parser_concat_to_echobuffer( ESIParser *parser, char ch )
 {
   parser->echobuffer_index++;
 
   if( parser->echobuffer_allocated <= parser->echobuffer_index ){
-    // double the echobuffer size
+    /* double the echobuffer size 
+     * we're getting some crazy input if this case ever happens
+     */
     parser->echobuffer_allocated *= 2;
     parser->echobuffer = (char*)realloc( parser->echobuffer, parser->echobuffer_allocated );
   }
   parser->echobuffer[parser->echobuffer_index] = ch;
 //  debug_string( "echo buffer", parser->echobuffer, parser->echobuffer_index+1 );
 }
-
+/*
+ * the mark boundary is not always going to be exactly on the attribute or tag name boundary
+ * this trims characters from the left to right, advancing *ptr and reducing *len
+ */
 static void ltrim_pointer( const char **ptr, const char *bounds, size_t *len )
 {
   // remove any spaces or = at the before the value
-  while( isspace( **ptr ) ||
+  while( (isspace( **ptr ) ||
          **ptr == '=' ||
          **ptr == '"' ||
          **ptr == '<' ||
-         **ptr == '\'' &&
-         *len > 0 &&
-         (*ptr != bounds) ){
+         **ptr == '\'' ) && (*len > 0) && (*ptr != bounds) ){
     (*ptr)++;
     (*len)--;
   }
 }
+/*
+ * similar to ltrim_pointer, this walks from bounds to *ptr, reducing *len 
+ */
+static void rtrim_pointer( const char **ptr, const char *bounds, size_t *len )
+{
+  bounds = (*ptr+(*len-1));
+  // remove any spaces or = at the before the value
+  while( (isspace( *bounds ) ||
+         *bounds == '=' ||
+         *bounds == '"' ||
+         *bounds == '>' ||
+         *bounds == '\'') && (*len > 0) && (*ptr != bounds) ){
+    bounds--;
+    (*len)--;
+  }
+}
 
 %%{
   machine esi;
@@ -94,53 +151,84 @@ static void ltrim_pointer( const char **ptr, const char *bounds, size_t *len )
   action see_start_tag {
     parser->tag_text = parser->mark+1;
     parser->tag_text_length = p - (parser->mark+1);
-//    debug_string( "have esi start", parser->tag_text, parser->tag_text_length );
     parser->mark = p;
-    //debug_string( "see_start_tag", p, 1 );
   }
 
+  # detected an inline tag end, sends the start tag and end tag callback
   action see_end_tag {
-    //debug_string( "see_end_tag: before parsed esi tag", parser->tag_text, parser->tag_text_length );
+    /* trim the tag text */
     ltrim_pointer( &(parser->tag_text), p, &(parser->tag_text_length) );
-    //debug_string( "see_end_tag: after parsed esi tag", parser->tag_text, parser->tag_text_length );
+    rtrim_pointer( &(parser->tag_text), p, &(parser->tag_text_length) );
+
+    /* send the start tag and end tag message */
+    esi_parser_flush_output( parser );
     parser->start_tag_handler( data, parser->tag_text, parser->tag_text_length, parser->attributes, parser->user_data );
-    //debug_string( "see_end_tag: after start parsed esi tag", parser->tag_text, parser->tag_text_length );
+    esi_parser_flush_output( parser );
     parser->end_tag_handler( data, parser->tag_text, parser->tag_text_length, parser->user_data );
-    //debug_string( "see_end_tag: after end parsed esi tag", parser->tag_text, parser->tag_text_length );
+    esi_parser_flush_output( parser );
+
+    /* mark the position */
     parser->tag_text = NULL;
     parser->tag_text_length = 0;
     parser->mark = p;
-    //debug_string( "see_end_tag", p, 1 );
+
+    /* clear out the echo buffer */
+    esi_parser_echobuffer_clear( parser );
   }
-  
+
+  # block tag start, with attributes
   action see_block_start_with_attributes {
-//    debug_string( "parsed esi tag", parser->tag_text, parser->tag_text_length );
+    /* trim tag text */
     ltrim_pointer( &(parser->tag_text), p, &(parser->tag_text_length) );
+    rtrim_pointer( &(parser->tag_text), p, &(parser->tag_text_length) );
+    
+    /* send the start and end tag message */
+    esi_parser_flush_output( parser );
     parser->start_tag_handler( data, parser->tag_text, parser->tag_text_length, parser->attributes, parser->user_data );
+    esi_parser_flush_output( parser );
+
     parser->tag_text = NULL;
     parser->tag_text_length = 0;
+    parser->mark = p;
+    
+    /* clear out the echo buffer */
+    esi_parser_echobuffer_clear( parser );
   }
-
+  
+  # see an attribute key, /foo\s*=/
   action see_attribute_key {
+    /* save the attribute  key start */
     parser->attr_key = parser->mark;
+    /* compute the length of the key */
     parser->attr_key_length = p - parser->mark;
+    /* save the position following the key */
     parser->mark = p;
+
+    /* trim the attribute key */
     ltrim_pointer( &(parser->attr_key), p, &(parser->attr_key_length) );
-//    debug_string( "\tattribute key", parser->attr_key, parser->attr_key_length );
-    //debug_string( "see_attribute_key", p, 1 );
+    rtrim_pointer( &(parser->attr_key), p, &(parser->attr_key_length) );
   }
 
+  # see an attribute value, aprox ~= /['"].*['"]/
   action see_attribute_value {
     ESIAttribute *attr;
+
+    /* save the attribute value start */
     parser->attr_value = parser->mark;
+    /* compute the length of the value */
     parser->attr_value_length = p - parser->mark;
+    /* svae the position following the value */
     parser->mark = p;
-
+    
+    /* trim the attribute value */
     ltrim_pointer( &(parser->attr_value), p, &(parser->attr_value_length) );
+    rtrim_pointer( &(parser->attr_value), p, &(parser->attr_value_length) );
 
-    // allocate a new attribute
+    /* using the attr_key and attr_value, allocate a new attribute object */
     attr = esi_attribute_new( parser->attr_key, parser->attr_key_length, 
                               parser->attr_value, parser->attr_value_length );
+
+    /* add the new attribute to the list of attributes */
     if( parser->attributes ){
       parser->last->next = attr;
       parser->last = attr;
@@ -148,49 +236,67 @@ static void ltrim_pointer( const char **ptr, const char *bounds, size_t *len )
     else{
       parser->last = parser->attributes = attr;
     }
-//    debug_string( "\tattribute value", parser->attr_value, parser->attr_value_length );
-    //debug_string( "see_attribute_value", p, 1 );
   }
 
+  # simple block start tag detected, e.g. <esi:try> no attributes
   action block_start_tag {
+
     parser->tag_text = parser->mark;
     parser->tag_text_length = p - parser->mark;
+
     parser->mark = p;
+
     ltrim_pointer( &(parser->tag_text), p, &(parser->tag_text_length) );
+    rtrim_pointer( &(parser->tag_text), p, &(parser->tag_text_length) );
+
+    esi_parser_flush_output( parser );
     parser->start_tag_handler( data, parser->tag_text, parser->tag_text_length, NULL, parser->user_data );
-    //debug_string( "block_start_tag", p, 1 );
+    esi_parser_flush_output( parser );
+
+    esi_parser_echobuffer_clear( parser );
   }
 
+  # block end tag detected, e.g. </esi:try>
   action block_end_tag {
+    /* offset by 2 to account for the </ characters */
     parser->tag_text = parser->mark+2;
     parser->tag_text_length = p - (parser->mark+2);
-    //printf( "mark: [%s], p: [%s], length: %d\n", parser->mark, p, parser->tag_text_length );
+
     parser->mark = p;
-    //debug_string( "block_end_tag: before parsed esi tag", parser->tag_text, parser->tag_text_length );
+    
     ltrim_pointer( &(parser->tag_text), p, &(parser->tag_text_length) );
-    //debug_string( "block_end_tag: after parsed esi tag", parser->tag_text, parser->tag_text_length );
+    rtrim_pointer( &(parser->tag_text), p, &(parser->tag_text_length) );
+    
+    esi_parser_flush_output( parser );
     parser->end_tag_handler( data, parser->tag_text, parser->tag_text_length, parser->user_data );
+    esi_parser_flush_output( parser );
+
+    esi_parser_echobuffer_clear( parser );
   }
 
+  # process each character in the input stream for output
   action echo {
+    //printf( "[%c:%d],", *p, cs );
     switch( cs ){
-    case 0:
-      if( parser->prev_state != 12 && parser->prev_state != 7 ){
-        if( parser->echobuffer && (parser->prev_state != (esi_en_main +1)) && parser->prev_state != 60 ){
-          // stream echobuffer
+    case 0: /* non matching state */
+      if( parser->prev_state != 12 && parser->prev_state != 7 ){ /* states following a possible end state for a tag */
+        if( parser->echobuffer && parser->echobuffer_index != -1 ){
+          /* send the echo buffer */
           esi_parser_echo_buffer( parser );
         }
-//    printf( "[%d: %c]", cs, *p );
-        // stream the current character
+        /* send the current character */
         esi_parser_echo_char( parser, *p );
       }
-      // clear the echobuffer
+      /* clear the echo buffer */
       esi_parser_echobuffer_clear( parser );
       break;
     default:
-      // append characters to echobuffer
+      /* append to the echo buffer */
       esi_parser_concat_to_echobuffer( parser, *p );
     }
+    /* save the previous state, necessary for end case detection such as />  and </esi:try>  the trailing > character
+      is state 12 and 7
+    */
     parser->prev_state = cs;
   }
 
@@ -199,6 +305,7 @@ static void ltrim_pointer( const char **ptr, const char *bounds, size_t *len )
 
 %%write data;
 
+/* dup the string up to len */
 char *esi_strndup( const char *str, size_t len )
 {
   char *s = (char*)malloc(sizeof(char)*(len+1));
@@ -260,7 +367,7 @@ ESIParser *esi_parser_new()
   parser->overflow_data = NULL;
 
   /* allocate 1024 bytes for the echobuffer */
-  parser->echobuffer_allocated = 1024;
+  parser->echobuffer_allocated = ESI_OUTPUT_BUFFER_SIZE; /* NOTE: change this value, to reduce memory consumtion or allocations */
   parser->echobuffer_index = -1;
   parser->echobuffer = (char*)malloc(sizeof(char)*parser->echobuffer_allocated);
 
@@ -271,6 +378,9 @@ ESIParser *esi_parser_new()
   parser->end_tag_handler = esi_parser_default_end_cb;
   parser->output_handler = esi_parser_default_output_cp;
 
+  parser->output_buffer_size = 0;
+  memset( parser->output_buffer, 0, ESI_OUTPUT_BUFFER_SIZE );
+
   return parser;
 }
 void esi_parser_free( ESIParser *parser )
@@ -304,14 +414,22 @@ static int compute_offset( const char *mark, const char *data )
   return -1;
 }
 
+/* accept an arbitrary length string buffer
+ * when this methods exits it determines if an end state was reached
+ * if no end state was reached it saves the full input into an internal buffer
+ * when invoked next, it reuses that internable buffer copying all pointers into the 
+ * newly allocated buffer. if it exits in a terminal state, e.g. 0 then it will dump these buffers
+ */
 int esi_parser_execute( ESIParser *parser, const char *data, size_t length )
 {
   int cs = parser->cs;
   const char *p = data;
+  const char *eof = NULL; // ragel 6.x compat
   const char *pe = data + length;
 
   if( length == 0 ){ return cs; }
 
+  /* there's an existing overflow buffer data append the new data to the existing data */
   if( parser->overflow_data && parser->overflow_data_size > 0 ){
 
     // recompute mark, tag_text, attr_key, and attr_value since they all exist within overflow_data
@@ -320,13 +438,11 @@ int esi_parser_execute( ESIParser *parser, const char *data, size_t length )
     int attr_key_offset = compute_offset( parser->attr_key, parser->overflow_data );
     int attr_value_offset = compute_offset( parser->attr_value, parser->overflow_data );
 
-//    debug_string("grow overflow buffer", parser->overflow_data, parser->overflow_data_size );
-
     parser->overflow_data = (char*)realloc( parser->overflow_data, sizeof(char)*(parser->overflow_data_size+length) );
     memcpy( parser->overflow_data+parser->overflow_data_size, data, length );
 
     p = parser->overflow_data + parser->overflow_data_size;
-    
+ 
     // in our new memory space mark will now be
     parser->mark = ( mark_offset >= 0 ) ? parser->overflow_data + mark_offset : NULL;
     parser->tag_text = ( tag_text_offset >= 0 ) ? parser->overflow_data + tag_text_offset : NULL;
@@ -335,9 +451,9 @@ int esi_parser_execute( ESIParser *parser, const char *data, size_t length )
  
     data = parser->overflow_data;
     parser->overflow_data_size = length = length + parser->overflow_data_size;
+//    printf( "grow overflow data: %ld\n", parser->overflow_data_size );
     pe = data + length;
 
-//    debug_string( "overflow", parser->overflow_data, parser->overflow_data_size );
   }
 
   if( !parser->mark ){
@@ -345,7 +461,6 @@ int esi_parser_execute( ESIParser *parser, const char *data, size_t length )
   }
 
 //  printf( "cs: %d, ", cs );
-//  debug_string( "data", data, length );
 
   %% write exec;
 
@@ -353,6 +468,7 @@ int esi_parser_execute( ESIParser *parser, const char *data, size_t length )
 
   if( cs != esi_start && cs != 0 ){
 
+    /* reached the end and we're not at a termination point save the buffer as overflow */
     if( !parser->overflow_data ){
       // recompute mark, tag_text, attr_key, and attr_value since they all exist within overflow_data
       int mark_offset = compute_offset( parser->mark, data );
@@ -364,6 +480,7 @@ int esi_parser_execute( ESIParser *parser, const char *data, size_t length )
       parser->overflow_data = (char*)malloc( sizeof( char ) * length );
       memcpy( parser->overflow_data, data, length );
       parser->overflow_data_size = length;
+//      printf( "allocate overflow data: %ld\n", parser->overflow_data_size );
 
       // in our new memory space mark will now be
       parser->mark = ( mark_offset >= 0 ) ? parser->overflow_data + mark_offset : NULL;
@@ -374,6 +491,7 @@ int esi_parser_execute( ESIParser *parser, const char *data, size_t length )
     }
 
   }else if( parser->overflow_data ){
+    /* dump the overflow buffer execution ended at a final state */
     free( parser->overflow_data );
     parser->overflow_data = NULL;
     parser->overflow_data_size = 0;
@@ -383,7 +501,7 @@ int esi_parser_execute( ESIParser *parser, const char *data, size_t length )
 }
 int esi_parser_finish( ESIParser *parser )
 {
-  %% write eof;
+  esi_parser_flush_output( parser );
   return 0;
 }