mongrel_esi 0.5.0 → 0.5.1

data/ext/esi/test/parser.rl

@@ -15,6 +15,8 @@ static void debug_string( const char *msg, const char *str, size_t len )
   printf( "%s :'%s'\n", msg, pstr );
   free( pstr );
 }
+#else
+#define debug_string(m,s,l)
 #endif
 
 /* define default callbacks */
@@ -93,7 +95,7 @@ static void esi_parser_concat_to_echobuffer( ESIParser *parser, char ch )
 {
   parser->echobuffer_index++;
 
-  if( parser->echobuffer_allocated <= parser->echobuffer_index ){
+  if( parser->echobuffer_allocated <= parser->echobuffer_index ) {
     /* double the echobuffer size 
      * we're getting some crazy input if this case ever happens
      */
@@ -114,7 +116,7 @@ static void ltrim_pointer( const char **ptr, const char *bounds, size_t *len )
          **ptr == '=' ||
          **ptr == '"' ||
          **ptr == '<' ||
-         **ptr == '\'' ) && (*len > 0) && (*ptr != bounds) ){
+         **ptr == '\'' ) && (*len > 0) && (*ptr != bounds) ) {
     (*ptr)++;
     (*len)--;
   }
@@ -229,11 +231,11 @@ static void rtrim_pointer( const char **ptr, const char *bounds, size_t *len )
                               parser->attr_value, parser->attr_value_length );
 
     /* add the new attribute to the list of attributes */
-    if( parser->attributes ){
+    if( parser->attributes ) {
       parser->last->next = attr;
       parser->last = attr;
     }
-    else{
+    else {
       parser->last = parser->attributes = attr;
     }
   }
@@ -277,7 +279,7 @@ static void rtrim_pointer( const char **ptr, const char *bounds, size_t *len )
   # process each character in the input stream for output
   action echo {
     //printf( "[%c:%d],", *p, cs );
-    switch( cs ){
+    switch( cs ) {
     case 0: /* non matching state */
       if( parser->prev_state != 12 && parser->prev_state != 7 ){ /* states following a possible end state for a tag */
         if( parser->echobuffer && parser->echobuffer_index != -1 ){
@@ -363,11 +365,14 @@ ESIParser *esi_parser_new()
   ESIParser *parser = (ESIParser*)malloc(sizeof(ESIParser));
   parser->cs = esi_start;
   parser->mark = NULL;
+  parser->tag_text = NULL;
+  parser->attr_key = NULL;
+  parser->attr_value = NULL;
   parser->overflow_data_size = 0;
   parser->overflow_data = NULL;
 
-  /* allocate 1024 bytes for the echobuffer */
-  parser->echobuffer_allocated = ESI_OUTPUT_BUFFER_SIZE; /* NOTE: change this value, to reduce memory consumtion or allocations */
+  /* allocate ESI_OUTPUT_BUFFER_SIZE bytes for the echobuffer */
+  parser->echobuffer_allocated = ESI_OUTPUT_BUFFER_SIZE;
   parser->echobuffer_index = -1;
   parser->echobuffer = (char*)malloc(sizeof(char)*parser->echobuffer_allocated);
 
@@ -408,12 +413,58 @@ int esi_parser_init( ESIParser *parser )
 
 static int compute_offset( const char *mark, const char *data )
 {
-  if( mark ){
+  if( mark ) {
     return mark - data;
   }
   return -1;
 }
 
+/*
+ * scans the data buffer for a start sequence /<$/, /<e$/, /<es$/, /<esi$/, /<esi:$/
+ * returns index of if start sequence found else returns -1
+ */
+static int
+esi_parser_scan_for_start( ESIParser *parser, const char *data, size_t length )
+{
+  size_t i, f = -2, s = -2;
+  char ch;
+
+  for( i = 0; i < length; ++i ) {
+    ch = data[i];
+    switch( ch ) {
+    case '<':
+      f = s = i;
+      break;
+    case '/':
+      if( s == (i-1) && f != -2 ) { s = i; }
+      break;
+    case 'e':
+      if( s == (i-1) && f != -2 ) { s = i; }
+      break;
+    case 's':
+      if( s == (i-1) && f != -2 ) { s = i; }
+      break;
+    case 'i':
+      if( s == (i-1) && f != -2 ) { s = i; }
+      break;
+    case ':':
+      if( s == (i-1) && f != -2 ) { s = i; return f; }
+      break;
+    default:
+      f = s = -2;
+      break;
+    }
+  }
+
+  // if s and f are still valid at end of input return f
+  if( f != -2 && s != -2 ) {
+    return f;
+  }
+  else {
+    return -1;
+  }
+}
+
 /* accept an arbitrary length string buffer
  * when this methods exits it determines if an end state was reached
  * if no end state was reached it saves the full input into an internal buffer
@@ -426,11 +477,23 @@ int esi_parser_execute( ESIParser *parser, const char *data, size_t length )
   const char *p = data;
   const char *eof = NULL; // ragel 6.x compat
   const char *pe = data + length;
+  int pindex;
 
   if( length == 0 ){ return cs; }
 
+  /* scan data for any '<esi:' start sequences, /<$/, /<e$/, /<es$/, /<esi$/, /<esi:$/ */
+  if( cs == esi_start ) { 
+    pindex = esi_parser_scan_for_start( parser, data, length );
+    if( pindex == -1 ) {
+      for( pindex = 0; pindex < length; ++pindex ) {
+        esi_parser_echo_char( parser, data[pindex] );
+      }
+      return cs;
+    }
+  }
+
   /* there's an existing overflow buffer data append the new data to the existing data */
-  if( parser->overflow_data && parser->overflow_data_size > 0 ){
+  if( parser->overflow_data && parser->overflow_data_size > 0 ) {
 
     // recompute mark, tag_text, attr_key, and attr_value since they all exist within overflow_data
     int mark_offset = compute_offset( parser->mark, parser->overflow_data );
@@ -456,7 +519,7 @@ int esi_parser_execute( ESIParser *parser, const char *data, size_t length )
 
   }
 
-  if( !parser->mark ){
+  if( !parser->mark ) {
     parser->mark = p;
   }
 
@@ -466,7 +529,7 @@ int esi_parser_execute( ESIParser *parser, const char *data, size_t length )
 
   parser->cs = cs;
 
-  if( cs != esi_start && cs != 0 ){
+  if( cs != esi_start && cs != 0 ) {
 
     /* reached the end and we're not at a termination point save the buffer as overflow */
     if( !parser->overflow_data ){
@@ -477,10 +540,16 @@ int esi_parser_execute( ESIParser *parser, const char *data, size_t length )
       int attr_value_offset = compute_offset( parser->attr_value, data );
       //debug_string( "mark before move", parser->mark, 1 );
 
-      parser->overflow_data = (char*)malloc( sizeof( char ) * length );
+      if( ESI_OUTPUT_BUFFER_SIZE > length ) {
+        parser->echobuffer_allocated = ESI_OUTPUT_BUFFER_SIZE;
+      }
+      else {
+        parser->echobuffer_allocated = length;
+      }
+      parser->overflow_data = (char*)malloc( sizeof( char ) * parser->echobuffer_allocated );
       memcpy( parser->overflow_data, data, length );
       parser->overflow_data_size = length;
-//      printf( "allocate overflow data: %ld\n", parser->overflow_data_size );
+      //printf( "allocate overflow data: %ld\n", parser->echobuffer_allocated );
 
       // in our new memory space mark will now be
       parser->mark = ( mark_offset >= 0 ) ? parser->overflow_data + mark_offset : NULL;
@@ -490,7 +559,7 @@ int esi_parser_execute( ESIParser *parser, const char *data, size_t length )
       //if( parser->mark ){ debug_string( "mark after  move", parser->mark, 1 ); } else { printf( "mark is now empty\n" ); }
     }
 
-  }else if( parser->overflow_data ){
+  }else if( parser->overflow_data ) {
     /* dump the overflow buffer execution ended at a final state */
     free( parser->overflow_data );
     parser->overflow_data = NULL;

data/ext/esi/test/sp.c

@@ -0,0 +1,125 @@
+/*
+ * This is an attempt at a custom fast parser for ESI
+ *
+ * call backs
+ *
+ * esi_output
+ * esi_start
+ * esi_end
+ *
+ */
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+
+typedef int (*esi_start_cb)(const char *tag_str, size_t tag_length, void *ptr);
+typedef int (*esi_end_cb)(const char *tag_str, size_t tag_length, void *ptr);
+typedef int (*esi_output_cb)(const char *data, size_t data_length, void *ptr);
+
+#define ESI_BUFFER_SIZE 1024
+
+typedef struct _ESIParser {
+  esi_start_cb start_cb;
+  esi_end_cb end_cb;
+  esi_output_cb output_cb;
+  char buffer[ESI_BUFFER_SIZE];
+  char tag_buffer[ESI_BUFFER_SIZE];
+  int tag_size;
+
+  void *callptr;
+}ESIParser;
+
+ESIParser *esi_parser_new();
+void esi_parser_free( ESIParser *parser );
+
+int esi_parser_feed( ESIParser *parser, const char *data, size_t length );
+
+int esi_parser_finish( ESIParser *parser );
+
+static int start_cb( const char *tag_str, size_t tag_length, void *ptr )
+{
+  return 0;
+}
+static int end_cb( const char *tag_str, size_t tag_length, void *ptr )
+{
+  return 0;
+}
+static int output_cb( const char *data, size_t data_length, void *ptr )
+{
+  return 0;
+}
+
+int main( int argc, char **argv )
+{
+  char *ptr = NULL;
+  const int buffer_size = 1024;
+  char buffer[buffer_size];
+  FILE *file = NULL;
+  ESIParser *parser = esi_parser_new();
+
+  parser->start_cb = start_cb;
+  parser->end_cb = end_cb;
+  parser->output_cb = output_cb;
+
+  file = fopen("sample.html","r");
+  if( !file ){ fprintf( stderr, "Failed to open sample.html\n" ); return 1; }
+
+  while( (ptr=fgets(buffer, buffer_size, file)) ) {
+    esi_parser_feed( parser, ptr, strlen(ptr) );
+  }
+
+  fclose(file);
+  esi_parser_free( parser );
+  return 0;
+}
+
+ESIParser *esi_parser_new()
+{
+  ESIParser *parser = (ESIParser*)calloc(1,sizeof(ESIParser));
+
+  return parser;
+}
+void esi_parser_free( ESIParser *parser )
+{
+  free( parser );
+}
+
+int esi_parser_feed( ESIParser *parser, const char *data, size_t length )
+{
+  size_t i;
+  size_t buffered = 0;
+
+  for( i = 0; (i+4) < length ; ++i ) {
+    if( data[i] == '<' &&
+        data[i+1] == 'e' &&
+        data[i+2] == 's' &&
+        data[i+3] == 'i' &&
+        data[i+4] == ':' ) { // start sequence
+      printf("start: %ld\n", i );
+    }
+    else if( data[i] == '<' &&
+        data[i+1] == '/' &&
+        data[i+2] == 'e' &&
+        data[i+3] == 's' &&
+        data[i+4] == 'i' &&
+        data[i+5] == ':' ) { // end sequence
+      printf("end: %ld\n", i );
+    }
+    else {
+      parser->buffer[buffered++] = data[i];
+      if( buffered == ESI_BUFFER_SIZE ) {
+        if( parser->output_cb ) {
+          parser->output_cb( parser->buffer, buffered, parser->callptr );
+        }
+        buffered = 0;
+      }
+    }
+  }
+  return buffered;
+}
+
+int esi_parser_finish( ESIParser *parser )
+{
+  return 0;
+}

data/ext/esi/test/test.c

@@ -35,6 +35,23 @@ TagInfo* tag_info_new( const char *name, size_t len, ESIAttribute *attributes )
   return ti;
 }
 
+static
+void tag_info_show( TagInfo *ti )
+{
+  ESIAttribute *attrs = ti->attributes;
+  if( attrs ) {
+    printf("%s{", ti->name );
+    while( attrs ) {
+      printf( "(%s=>%s),", attrs->name, attrs->value );
+      attrs = attrs->next;
+    }
+    printf("}\n");
+  }
+  else {
+    printf("%s\n", ti->name );
+  }
+}
+
 static
 void tag_info_free( TagInfo *ti )
 {
@@ -68,7 +85,7 @@ static int verify_match_string( const char *expr, const char *str_value, int lin
   status = regexec( &reg_expr, str_value, (size_t) 0, NULL, 0 );
   regfree( &reg_expr );
   if( status != 0 ){
-    printf( "No matching %s expression found in '%s' at %s:%d ", expr, str_value, test_name, line );
+    printf( "No matching %s expression found in '%s' at %s:%d \n", expr, str_value, test_name, line );
     return 1;
   }
   return 0;
@@ -181,7 +198,7 @@ static void feed_data( ESIParser *parser, const char *data )
 #define TEST_WITH_FILE(file)\
   int size = 0;\
   FILE *input = NULL;\
-  char buffer[16]; /* intentially using a small buffer size */\
+  char buffer[1024]; /* intentially using a small buffer size */\
   TEST_INIT\
 \
   input = fopen( file, "r" );\
@@ -190,7 +207,7 @@ static void feed_data( ESIParser *parser, const char *data )
     return;\
   }\
 \
-  while( (size = fread( buffer, sizeof(char), 15, input )) > 0 ){\
+  while( (size = fread( buffer, sizeof(char), 1023, input )) > 0 ){\
     esi_parser_execute( parser, buffer, size );\
   }\
 \
@@ -257,6 +274,36 @@ static void test_simple_parser_input( ESIParser *parser )
   TEST_FINISH
 }
 
+static void test_small_buffer( ESIParser *parser )
+{
+  const int input_size = 2;
+  const char *input = "<p>some input</p><esi:include />some more input\nsome input<esi:include timeout='10' src='hello'/>some more input";
+  int i, len = strlen(input);
+
+  TEST_INIT
+
+  for( i = 0; i < len; i += input_size ) {
+    if( i + input_size <= len ) {
+      esi_parser_execute( parser, (input+i), input_size );
+    }
+    else {
+      printf("using: %d\n", len - i );
+      esi_parser_execute( parser, (input+i), len - i );
+    }
+  }
+
+  TEST_PREPARE_ASSERTIONS
+
+  ASSERT_EQUAL( "<p>some input</p>some more input\nsome inputsome more input", output );
+  ASSERT_TRUE( start_tag_count == 2 );
+  ASSERT_TRUE( end_tag_count == 2 );
+  
+  ASSERT_TAG_NAMES( start_tags )
+  ASSERT_TAG_NAMES( end_tags )
+
+  TEST_FINISH
+}
+
 static void test_chunked_input( ESIParser *parser )
 {
   TEST_INIT
@@ -313,7 +360,8 @@ static void test_sample_input( ESIParser *parser )
 
   printf( "start_tag_count: %d\n", start_tag_count );
   printf( "end_tag_count: %d\n", end_tag_count );
-*/
+  */
+
   ASSERT_TRUE( start_tag_count == 13 );
   ASSERT_TRUE( end_tag_count == 13 );
 
@@ -361,13 +409,89 @@ static void test_large_file( ESIParser *parser )
 
   TEST_FINISH
 }
-  
-int main( int argc, char **argv )
+
+static void test_sample1( ESIParser *parser )
+{
+  TEST_WITH_FILE( "sample1.html" );
+
+  TEST_PREPARE_ASSERTIONS
+
+//  ASSERT_MATCH("YYY", output);
+  //ASSERT_NO_MATCH("failed", output );
+  ASSERT_NO_MATCH("<esi:", output);
+  ASSERT_MATCH("<html", output);
+  ASSERT_MATCH("</html>", output);
+  ASSERT_MATCH("line 1: <pre>", output);
+  ASSERT_MATCH("line 2: <pre>", output);
+  ASSERT_MATCH("line 3: <pre>", output);
+  ASSERT_MATCH("line 4: <pre>", output);
+  ASSERT_MATCH("line 5: <pre>", output);
+  ASSERT_MATCH("line 6: <pre>", output);
+  ASSERT_MATCH("line 7: <pre>", output);
+  ASSERT_MATCH("line 8: <pre>", output);
+  ASSERT_MATCH("line 9: <pre>", output);
+
+
+  /*
+  TagInfo *ptr = start_tags;
+  while( ptr  ) { 
+    tag_info_show( ptr );
+    ptr = ptr->next;
+  }
+  printf( "start_tag_count: %d\n", start_tag_count );
+  printf( "end_tag_count: %d\n", end_tag_count );
+  */
+ 
+
+  ASSERT_TRUE( start_tag_count == 30 );
+  ASSERT_TRUE( end_tag_count == 30 );
+
+  ASSERT_TAG_NAMES( start_tags )
+  ASSERT_TAG_NAMES( end_tags )
+
+  TEST_FINISH
+}
+#if 0
+static void test_sample1_with_chunking( ESIParser *parser )
+{
+  TEST_WITH_FILE( "sample1.html" );
+
+  TEST_PREPARE_ASSERTIONS
+
+//  ASSERT_MATCH("YYY", output);
+  ASSERT_NO_MATCH("failed", output );
+  ASSERT_NO_MATCH("<esi:", output);
+
+  /*TagInfo *ptr = start_tags;
+  while( ptr  ) { 
+    tag_info_show( ptr );
+    ptr = ptr->next;
+  }
+  printf( "start_tag_count: %d\n", start_tag_count );
+  printf( "end_tag_count: %d\n", end_tag_count );
+  */
+
+  ASSERT_TRUE( start_tag_count == 23 );
+  ASSERT_TRUE( end_tag_count == 16 );
+
+  ASSERT_TAG_NAMES( start_tags )
+  ASSERT_TAG_NAMES( end_tags )
+
+  TEST_FINISH
+}
+#endif
+
+static void test_large_chunked_file( ESIParser *parser );
+
+static void run_parser_through_all()
 {
   ESIParser *parser = esi_parser_new();
+  printf( "%s\n", __FUNCTION__ );
 
   test_simple_parser_input( parser );
 
+  test_small_buffer( parser );
+
   test_chunked_input( parser );
 
   test_sample_input( parser );
@@ -376,7 +500,75 @@ int main( int argc, char **argv )
   
   test_large_file( parser );
 
+  test_large_chunked_file( parser );
+
+  test_sample1( parser );
+
   esi_parser_free( parser );
+}
 
+int main( int argc, char **argv )
+{
+  run_parser_through_all();
   return 0;
 }
+
+#define ESI_LARGE_FILE_CHUNKED "chunks.txt" 
+static void test_large_chunked_file( ESIParser *parser )
+{
+  int size = 0;
+  FILE *input = NULL;
+  char *buffer = NULL;
+  char *chunk_start = NULL;
+  char *chunk_end = NULL;
+  char *buffer_end = NULL;
+
+  TEST_INIT
+  
+//  printf( "\n" );
+
+  input = fopen( ESI_LARGE_FILE_CHUNKED, "r" );
+  if( !input ){
+    printf( "Failed to open %s\n", ESI_LARGE_FILE_CHUNKED );
+    return;
+  }
+
+  fstat( fileno(input), &st );
+
+  buffer = (char*)malloc(sizeof(char)*st.st_size);
+
+  size = fread( buffer, sizeof(char), st.st_size, input );
+  if( size != st.st_size ) {
+    printf( "Read error\n" );
+    return;
+  }
+  buffer_end = buffer + size;
+
+  for( chunk_start = buffer; chunk_start != buffer_end; ) {
+    while( *chunk_start != '{' && chunk_start != buffer_end ) { ++chunk_start; }
+    if( chunk_start == buffer_end ){ break; }
+    chunk_end = chunk_start;
+    while( *chunk_end != '}' && chunk_end != buffer_end ) { ++chunk_end; }
+    ++chunk_start;
+    if( chunk_start > buffer_end ){ break; }
+
+    if( (chunk_end - chunk_start) > 0 ) {
+      esi_parser_execute( parser, chunk_start, chunk_end - chunk_start );
+      chunk_start = chunk_end;
+    }
+  }
+
+  fclose( input );
+  free( buffer );
+
+  TEST_PREPARE_ASSERTIONS
+
+  ASSERT_NO_MATCH("<esi:", output);
+
+  ASSERT_MATCH( "</html>", output );
+
+  ASSERT_TAG_NAMES( start_tags )
+  ASSERT_TAG_NAMES( end_tags )
+
+  TEST_FINISH
+}