herb 0.5.0 → 0.6.0

data/src/include/errors.h

@@ -1,5 +1,5 @@
 // NOTE: This file is generated by the templates/template.rb script and should not
-// be modified manually. See /Users/marcoroth/Development/herb-release-0.5.0/templates/src/include/errors.h.erb
+// be modified manually. See /Users/marcoroth/Development/herb-release-0.6.0/templates/src/include/errors.h.erb
 
 #ifndef HERB_ERRORS_H
 #define HERB_ERRORS_H

data/src/include/herb.h

@@ -5,6 +5,7 @@
 #include "ast_node.h"
 #include "buffer.h"
 #include "extract.h"
+#include "parser.h"
 
 #include <stdint.h>
 
@@ -18,7 +19,7 @@ void herb_lex_json_to_buffer(const char* source, buffer_T* output);
 array_T* herb_lex(const char* source);
 array_T* herb_lex_file(const char* path);
 
-AST_DOCUMENT_NODE_T* herb_parse(const char* source);
+AST_DOCUMENT_NODE_T* herb_parse(const char* source, parser_options_T* options);
 
 const char* herb_version(void);
 const char* herb_prism_version(void);

data/src/include/lexer_peek_helpers.h

@@ -2,13 +2,28 @@
 #define HERB_LEXER_PEEK_HELPERS_H
 
 #include "lexer_struct.h"
+#include "token_struct.h"
 
 #include <stdbool.h>
 #include <stdio.h>
 #include <stdlib.h>
 
+typedef struct {
+  size_t position;
+  size_t line;
+  size_t column;
+  size_t previous_position;
+  size_t previous_line;
+  size_t previous_column;
+  char current_character;
+  lexer_state_T state;
+} lexer_state_snapshot_T;
+
 char lexer_peek(const lexer_T* lexer, int offset);
 bool lexer_peek_for_doctype(const lexer_T* lexer, int offset);
+bool lexer_peek_for_xml_declaration(const lexer_T* lexer, int offset);
+bool lexer_peek_for_cdata_start(const lexer_T* lexer, int offset);
+bool lexer_peek_for_cdata_end(const lexer_T* lexer, int offset);
 
 bool lexer_peek_for_html_comment_start(const lexer_T* lexer, int offset);
 bool lexer_peek_for_html_comment_end(const lexer_T* lexer, int offset);
@@ -20,4 +35,10 @@ bool lexer_peek_erb_end(const lexer_T* lexer, int offset);
 
 char lexer_backtrack(const lexer_T* lexer, int offset);
 
+bool lexer_peek_for_token_type_after_whitespace(lexer_T* lexer, token_type_T token_type);
+bool lexer_peek_for_close_tag_start(const lexer_T* lexer, int offset);
+
+lexer_state_snapshot_T lexer_save_state(lexer_T* lexer);
+void lexer_restore_state(lexer_T* lexer, lexer_state_snapshot_T snapshot);
+
 #endif

data/src/include/parser.h

@@ -15,15 +15,20 @@ typedef enum {
 
 typedef enum { PARSER_STATE_DATA, PARSER_STATE_FOREIGN_CONTENT } parser_state_T;
 
+typedef struct PARSER_OPTIONS_STRUCT {
+  bool track_whitespace;
+} parser_options_T;
+
 typedef struct PARSER_STRUCT {
   lexer_T* lexer;
   token_T* current_token;
   array_T* open_tags_stack;
   parser_state_T state;
   foreign_content_type_T foreign_content_type;
+  parser_options_T* options;
 } parser_T;
 
-parser_T* parser_init(lexer_T* lexer);
+parser_T* parser_init(lexer_T* lexer, parser_options_T* options);
 
 AST_DOCUMENT_NODE_T* parser_parse(parser_T* parser);
 

data/src/include/token_struct.h

@@ -10,7 +10,11 @@ typedef enum {
   TOKEN_NEWLINE,    // \n
   TOKEN_IDENTIFIER,
 
-  TOKEN_HTML_DOCTYPE, // <!DOCTYPE, <!doctype, <!DoCtYpE, <!dOcTyPe
+  TOKEN_HTML_DOCTYPE,        // <!DOCTYPE, <!doctype, <!DoCtYpE, <!dOcTyPe
+  TOKEN_XML_DECLARATION,     // <?xml
+  TOKEN_XML_DECLARATION_END, // ?>
+  TOKEN_CDATA_START,         // <![CDATA[
+  TOKEN_CDATA_END,           // ]]>
 
   TOKEN_HTML_TAG_START,       // <
   TOKEN_HTML_TAG_START_CLOSE, // </
@@ -29,6 +33,7 @@ typedef enum {
   TOKEN_EQUALS,      // =
   TOKEN_QUOTE,       // ", '
   TOKEN_BACKTICK,    // `
+  TOKEN_BACKSLASH,   // backslash
   TOKEN_DASH,        // -
   TOKEN_UNDERSCORE,  // _
   TOKEN_EXCLAMATION, // !

data/src/include/version.h

@@ -1,6 +1,6 @@
 #ifndef HERB_VERSION_H
 #define HERB_VERSION_H
 
-#define HERB_VERSION "0.5.0"
+#define HERB_VERSION "0.6.0"
 
 #endif

data/src/lexer.c

@@ -290,13 +290,21 @@ token_T* lexer_next_token(lexer_T* lexer) {
         return lexer_advance_with_next(lexer, strlen("<!DOCTYPE"), TOKEN_HTML_DOCTYPE);
       }
 
+      if (lexer_peek_for_xml_declaration(lexer, 0)) {
+        return lexer_advance_with_next(lexer, strlen("<?xml"), TOKEN_XML_DECLARATION);
+      }
+
+      if (lexer_peek_for_cdata_start(lexer, 0)) {
+        return lexer_advance_with_next(lexer, strlen("<![CDATA["), TOKEN_CDATA_START);
+      }
+
       if (isalnum(lexer_peek(lexer, 1))) { return lexer_advance_current(lexer, TOKEN_HTML_TAG_START); }
 
       if (lexer_peek_for_html_comment_start(lexer, 0)) {
         return lexer_advance_with(lexer, "<!--", TOKEN_HTML_COMMENT_START);
       }
 
-      if (lexer_peek(lexer, 1) == '/' && isalnum(lexer_peek(lexer, 2))) {
+      if (lexer_peek_for_close_tag_start(lexer, 0)) {
         return lexer_advance_with(lexer, "</", TOKEN_HTML_TAG_START_CLOSE);
       }
 
@@ -308,11 +316,21 @@ token_T* lexer_next_token(lexer_T* lexer) {
       return token ? token : lexer_advance_current(lexer, TOKEN_SLASH);
     }
 
+    case '?': {
+      token_T* token = lexer_match_and_advance(lexer, "?>", TOKEN_XML_DECLARATION_END);
+      return token ? token : lexer_advance_current(lexer, TOKEN_CHARACTER);
+    }
+
     case '-': {
       token_T* token = lexer_match_and_advance(lexer, "-->", TOKEN_HTML_COMMENT_END);
       return token ? token : lexer_advance_current(lexer, TOKEN_DASH);
     }
 
+    case ']': {
+      token_T* token = lexer_match_and_advance(lexer, "]]>", TOKEN_CDATA_END);
+      return token ? token : lexer_advance_current(lexer, TOKEN_CHARACTER);
+    }
+
     case '>': return lexer_advance_current(lexer, TOKEN_HTML_TAG_END);
     case '_': return lexer_advance_current(lexer, TOKEN_UNDERSCORE);
     case ':': return lexer_advance_current(lexer, TOKEN_COLON);
@@ -326,6 +344,7 @@ token_T* lexer_next_token(lexer_T* lexer) {
     case '"':
     case '\'': return lexer_advance_current(lexer, TOKEN_QUOTE);
     case '`': return lexer_advance_current(lexer, TOKEN_BACKTICK);
+    case '\\': return lexer_advance_current(lexer, TOKEN_BACKSLASH);
 
     default: {
       if (isalnum(lexer->current_character)) { return lexer_parse_identifier(lexer); }

data/src/lexer_peek_helpers.c

@@ -1,6 +1,8 @@
 #include "include/lexer_peek_helpers.h"
+#include "include/lexer.h"
 #include "include/lexer_struct.h"
 #include "include/macros.h"
+#include "include/token.h"
 
 #include <ctype.h>
 #include <stdbool.h>
@@ -31,6 +33,18 @@ bool lexer_peek_for_doctype(const lexer_T* lexer, const int offset) {
   return lexer_peek_for(lexer, offset, "<!DOCTYPE", true);
 }
 
+bool lexer_peek_for_xml_declaration(const lexer_T* lexer, const int offset) {
+  return lexer_peek_for(lexer, offset, "<?xml", true);
+}
+
+bool lexer_peek_for_cdata_start(const lexer_T* lexer, const int offset) {
+  return lexer_peek_for(lexer, offset, "<![CDATA[", false);
+}
+
+bool lexer_peek_for_cdata_end(const lexer_T* lexer, const int offset) {
+  return lexer_peek_for(lexer, offset, "]]>", false);
+}
+
 bool lexer_peek_for_html_comment_start(const lexer_T* lexer, const int offset) {
   return lexer_peek_for(lexer, offset, "<!--", false);
 }
@@ -57,3 +71,66 @@ bool lexer_peek_erb_end(const lexer_T* lexer, const int offset) {
     || lexer_peek_erb_percent_close_tag(lexer, offset)
   );
 }
+
+bool lexer_peek_for_token_type_after_whitespace(lexer_T* lexer, token_type_T token_type) {
+  size_t saved_position = lexer->current_position;
+  size_t saved_line = lexer->current_line;
+  size_t saved_column = lexer->current_column;
+  char saved_character = lexer->current_character;
+
+  token_T* token = lexer_next_token(lexer);
+
+  while (token && (token->type == TOKEN_WHITESPACE || token->type == TOKEN_NEWLINE)) {
+    token_free(token);
+    token = lexer_next_token(lexer);
+  }
+
+  bool result = (token && token->type == token_type);
+
+  if (token) { token_free(token); }
+
+  lexer->current_position = saved_position;
+  lexer->current_line = saved_line;
+  lexer->current_column = saved_column;
+  lexer->current_character = saved_character;
+
+  return result;
+}
+
+bool lexer_peek_for_close_tag_start(const lexer_T* lexer, const int offset) {
+  if (lexer_peek(lexer, offset) != '<' || lexer_peek(lexer, offset + 1) != '/') { return false; }
+
+  int pos = offset + 2;
+
+  while (lexer_peek(lexer, pos) == ' ' || lexer_peek(lexer, pos) == '\t' || lexer_peek(lexer, pos) == '\n'
+         || lexer_peek(lexer, pos) == '\r') {
+    pos++;
+  }
+
+  char c = lexer_peek(lexer, pos);
+
+  return isalpha(c) || c == '_';
+}
+
+lexer_state_snapshot_T lexer_save_state(lexer_T* lexer) {
+  lexer_state_snapshot_T snapshot = { .position = lexer->current_position,
+                                      .line = lexer->current_line,
+                                      .column = lexer->current_column,
+                                      .previous_position = lexer->previous_position,
+                                      .previous_line = lexer->previous_line,
+                                      .previous_column = lexer->previous_column,
+                                      .current_character = lexer->current_character,
+                                      .state = lexer->state };
+  return snapshot;
+}
+
+void lexer_restore_state(lexer_T* lexer, lexer_state_snapshot_T snapshot) {
+  lexer->current_position = snapshot.position;
+  lexer->current_line = snapshot.line;
+  lexer->current_column = snapshot.column;
+  lexer->previous_position = snapshot.previous_position;
+  lexer->previous_line = snapshot.previous_line;
+  lexer->previous_column = snapshot.previous_column;
+  lexer->current_character = snapshot.current_character;
+  lexer->state = snapshot.state;
+}

data/src/main.c

@@ -63,7 +63,7 @@ int main(const int argc, char* argv[]) {
   clock_gettime(CLOCK_MONOTONIC, &start);
 
   if (strcmp(argv[1], "visit") == 0) {
-    AST_DOCUMENT_NODE_T* root = herb_parse(source);
+    AST_DOCUMENT_NODE_T* root = herb_parse(source, NULL);
     clock_gettime(CLOCK_MONOTONIC, &end);
 
     herb_analyze_parse_tree(root, source);
@@ -105,7 +105,7 @@ int main(const int argc, char* argv[]) {
   }
 
   if (strcmp(argv[1], "parse") == 0) {
-    AST_DOCUMENT_NODE_T* root = herb_parse(source);
+    AST_DOCUMENT_NODE_T* root = herb_parse(source, NULL);
     clock_gettime(CLOCK_MONOTONIC, &end);
 
     ast_pretty_print_node((AST_NODE_T*) root, 0, 0, &output);