herb 0.0.1 → 0.1.0

data/src/lexer.c

@@ -0,0 +1,284 @@
+#include "include/buffer.h"
+#include "include/lexer_peek_helpers.h"
+#include "include/token.h"
+#include "include/util.h"
+
+#include <ctype.h>
+#include <string.h>
+
+#define LEXER_STALL_LIMIT 5
+
+static size_t lexer_sizeof(void) {
+  return sizeof(struct LEXER_STRUCT);
+}
+
+static bool lexer_eof(const lexer_T* lexer) {
+  return lexer->current_character == '\0' || lexer->stalled;
+}
+
+static bool lexer_has_more_characters(const lexer_T* lexer) {
+  return lexer->current_position < lexer->source_length;
+}
+
+static bool lexer_stalled(lexer_T* lexer) {
+  if (lexer->last_position == lexer->current_position) {
+    lexer->stall_counter++;
+
+    if (lexer->stall_counter > LEXER_STALL_LIMIT) { lexer->stalled = true; }
+  } else {
+    lexer->stall_counter = 0;
+    lexer->last_position = lexer->current_position;
+  }
+
+  return lexer->stalled;
+}
+
+lexer_T* lexer_init(const char* source) {
+  if (source == NULL) { source = ""; }
+
+  lexer_T* lexer = calloc(1, lexer_sizeof());
+
+  lexer->state = STATE_DATA;
+
+  lexer->source = source;
+  lexer->source_length = strlen(source);
+  lexer->current_character = source[0];
+
+  lexer->current_line = 1;
+  lexer->current_column = 0;
+  lexer->current_position = 0;
+
+  lexer->previous_line = lexer->current_line;
+  lexer->previous_column = lexer->current_column;
+  lexer->previous_position = lexer->current_position;
+
+  lexer->stall_counter = 0;
+  lexer->last_position = 0;
+  lexer->stalled = false;
+
+  return lexer;
+}
+
+token_T* lexer_error(lexer_T* lexer, const char* message) {
+  char error_message[128];
+
+  snprintf(
+    error_message,
+    sizeof(error_message),
+    "[Lexer] Error: %s (character '%c', line %zu, col %zu)\n",
+    message,
+    lexer->current_character,
+    lexer->current_line,
+    lexer->current_column
+  );
+
+  return token_init(herb_strdup(error_message), TOKEN_ERROR, lexer);
+}
+
+static void lexer_advance(lexer_T* lexer) {
+  if (lexer_has_more_characters(lexer) && !lexer_eof(lexer)) {
+    if (!is_newline(lexer->current_character)) { lexer->current_column++; }
+
+    lexer->current_position++;
+    lexer->current_character = lexer->source[lexer->current_position];
+  }
+}
+
+static void lexer_advance_by(lexer_T* lexer, const size_t count) {
+  for (size_t i = 0; i < count; i++) {
+    lexer_advance(lexer);
+  }
+}
+
+static token_T* lexer_advance_with(lexer_T* lexer, const char* value, const token_type_T type) {
+  lexer_advance_by(lexer, strlen(value));
+  return token_init(value, type, lexer);
+}
+
+static token_T* lexer_advance_with_next(lexer_T* lexer, size_t count, token_type_T type) {
+  char* collected = malloc(count + 1);
+  if (!collected) { return NULL; }
+
+  for (size_t i = 0; i < count; i++) {
+    collected[i] = lexer->current_character;
+    lexer_advance(lexer);
+  }
+
+  collected[count] = '\0';
+
+  token_T* token = token_init(collected, type, lexer);
+  free(collected);
+
+  return token;
+}
+
+static token_T* lexer_advance_current(lexer_T* lexer, const token_type_T type) {
+  return lexer_advance_with(lexer, (char[]) { lexer->current_character, '\0' }, type);
+}
+
+static token_T* lexer_match_and_advance(lexer_T* lexer, const char* value, const token_type_T type) {
+  if (strncmp(lexer->source + lexer->current_position, value, strlen(value)) == 0) {
+    return lexer_advance_with(lexer, value, type);
+  }
+
+  return NULL;
+}
+
+// ===== Specialized Parsers
+
+static token_T* lexer_parse_whitespace(lexer_T* lexer) {
+  buffer_T buffer = buffer_new();
+
+  while (isspace(lexer->current_character) && lexer->current_character != '\n' && lexer->current_character != '\r'
+         && !lexer_eof(lexer)) {
+    buffer_append_char(&buffer, lexer->current_character);
+    lexer_advance(lexer);
+  }
+
+  token_T* token = token_init(buffer.value, TOKEN_WHITESPACE, lexer);
+
+  buffer_free(&buffer);
+
+  return token;
+}
+
+static token_T* lexer_parse_identifier(lexer_T* lexer) {
+  buffer_T buffer = buffer_new();
+
+  while ((isalnum(lexer->current_character) || lexer->current_character == '-' || lexer->current_character == '_'
+          || lexer->current_character == ':')
+         && !lexer_peek_for_html_comment_end(lexer, 0) && !lexer_eof(lexer)) {
+
+    buffer_append_char(&buffer, lexer->current_character);
+    lexer_advance(lexer);
+  }
+
+  token_T* token = token_init(buffer.value, TOKEN_IDENTIFIER, lexer);
+
+  buffer_free(&buffer);
+
+  return token;
+}
+
+// ===== ERB Parsing
+
+static token_T* lexer_parse_erb_open(lexer_T* lexer) {
+  const char* erb_patterns[] = { "<%==", "<%=", "<%#", "<%-", "<%%", "<%" };
+
+  lexer->state = STATE_ERB_CONTENT;
+
+  for (size_t i = 0; i < sizeof(erb_patterns) / sizeof(erb_patterns[0]); i++) {
+    token_T* match = lexer_match_and_advance(lexer, erb_patterns[i], TOKEN_ERB_START);
+    if (match) { return match; }
+  }
+
+  return lexer_error(lexer, "Unexpected ERB start");
+}
+
+static token_T* lexer_parse_erb_content(lexer_T* lexer) {
+  buffer_T buffer = buffer_new();
+
+  while (!lexer_peek_erb_end(lexer, 0)) {
+    if (lexer_eof(lexer)) {
+      return token_init(buffer.value, TOKEN_ERROR, lexer); // Handle unexpected EOF
+    }
+
+    buffer_append_char(&buffer, lexer->current_character);
+    lexer_advance(lexer);
+  }
+
+  lexer->state = STATE_ERB_CLOSE;
+
+  token_T* token = token_init(buffer.value, TOKEN_ERB_CONTENT, lexer);
+
+  buffer_free(&buffer);
+
+  return token;
+}
+
+static token_T* lexer_parse_erb_close(lexer_T* lexer) {
+  lexer->state = STATE_DATA;
+
+  if (lexer_peek_erb_percent_close_tag(lexer, 0)) { return lexer_advance_with(lexer, "%%>", TOKEN_ERB_END); }
+  if (lexer_peek_erb_dash_close_tag(lexer, 0)) { return lexer_advance_with(lexer, "-%>", TOKEN_ERB_END); }
+
+  return lexer_advance_with(lexer, "%>", TOKEN_ERB_END);
+}
+
+// ===== Tokenizing Function
+
+token_T* lexer_next_token(lexer_T* lexer) {
+  if (lexer_eof(lexer)) { return token_init("", TOKEN_EOF, lexer); }
+  if (lexer_stalled(lexer)) { return lexer_error(lexer, "Lexer stalled after 5 iterations"); }
+
+  if (lexer->state == STATE_ERB_CONTENT) { return lexer_parse_erb_content(lexer); }
+  if (lexer->state == STATE_ERB_CLOSE) { return lexer_parse_erb_close(lexer); }
+
+  if (lexer->current_character == '\r' && lexer_peek(lexer, 1) == '\n') {
+    return lexer_advance_with_next(lexer, 2, TOKEN_NEWLINE);
+  }
+  if (lexer->current_character == '\n') { return lexer_advance_current(lexer, TOKEN_NEWLINE); }
+  if (lexer->current_character == '\r') { return lexer_advance_current(lexer, TOKEN_NEWLINE); }
+
+  if (isspace(lexer->current_character)) { return lexer_parse_whitespace(lexer); }
+
+  if (lexer->current_character == '\xC2' && lexer_peek(lexer, 1) == '\xA0') {
+    return lexer_advance_with(lexer, "\xC2\xA0", TOKEN_NBSP);
+  }
+
+  switch (lexer->current_character) {
+    case '<': {
+      if (lexer_peek(lexer, 1) == '%') { return lexer_parse_erb_open(lexer); }
+
+      if (lexer_peek_for_doctype(lexer, 0)) {
+        return lexer_advance_with_next(lexer, strlen("<!DOCTYPE"), TOKEN_HTML_DOCTYPE);
+      }
+
+      if (isalnum(lexer_peek(lexer, 1))) { return lexer_advance_current(lexer, TOKEN_HTML_TAG_START); }
+
+      if (lexer_peek_for_html_comment_start(lexer, 0)) {
+        return lexer_advance_with(lexer, "<!--", TOKEN_HTML_COMMENT_START);
+      }
+
+      if (lexer_peek(lexer, 1) == '/' && isalnum(lexer_peek(lexer, 2))) {
+        return lexer_advance_with(lexer, "</", TOKEN_HTML_TAG_START_CLOSE);
+      }
+
+      return lexer_advance_current(lexer, TOKEN_LT);
+    }
+
+    case '/': {
+      token_T* token = lexer_match_and_advance(lexer, "/>", TOKEN_HTML_TAG_SELF_CLOSE);
+      return token ? token : lexer_advance_current(lexer, TOKEN_SLASH);
+    }
+
+    case '-': {
+      token_T* token = lexer_match_and_advance(lexer, "-->", TOKEN_HTML_COMMENT_END);
+      return token ? token : lexer_advance_current(lexer, TOKEN_DASH);
+    }
+
+    case '>': return lexer_advance_current(lexer, TOKEN_HTML_TAG_END);
+    case '_': return lexer_advance_current(lexer, TOKEN_UNDERSCORE);
+    case ':': return lexer_advance_current(lexer, TOKEN_COLON);
+    case ';': return lexer_advance_current(lexer, TOKEN_SEMICOLON);
+    case '&': return lexer_advance_current(lexer, TOKEN_AMPERSAND);
+    case '!': return lexer_advance_current(lexer, TOKEN_EXCLAMATION);
+    case '=': return lexer_advance_current(lexer, TOKEN_EQUALS);
+    case '%': return lexer_advance_current(lexer, TOKEN_PERCENT);
+
+    case '"':
+    case '\'': return lexer_advance_current(lexer, TOKEN_QUOTE);
+
+    default: {
+      if (isalnum(lexer->current_character)) { return lexer_parse_identifier(lexer); }
+
+      return lexer_advance_current(lexer, TOKEN_CHARACTER);
+    }
+  }
+}
+
+void lexer_free(lexer_T* lexer) {
+  if (lexer == NULL) { return; }
+
+  free(lexer);
+}

data/src/lexer_peek_helpers.c

@@ -0,0 +1,59 @@
+#include "include/lexer_peek_helpers.h"
+#include "include/lexer_struct.h"
+#include "include/macros.h"
+
+#include <ctype.h>
+#include <stdbool.h>
+
+char lexer_backtrack(const lexer_T* lexer, const int offset) {
+  return lexer->source[MAX(lexer->current_position - offset, 0)];
+}
+
+char lexer_peek(const lexer_T* lexer, const int offset) {
+  return lexer->source[MIN(lexer->current_position + offset, lexer->source_length)];
+}
+
+bool lexer_peek_for(const lexer_T* lexer, const int offset, const char* pattern, const bool case_insensitive) {
+  for (int index = 0; pattern[index]; index++) {
+    const char character = lexer_peek(lexer, offset + index);
+
+    if (case_insensitive) {
+      if (tolower(character) != tolower(pattern[index])) { return false; }
+    } else {
+      if (character != pattern[index]) { return false; }
+    }
+  }
+
+  return true;
+}
+
+bool lexer_peek_for_doctype(const lexer_T* lexer, const int offset) {
+  return lexer_peek_for(lexer, offset, "<!DOCTYPE", true);
+}
+
+bool lexer_peek_for_html_comment_start(const lexer_T* lexer, const int offset) {
+  return lexer_peek_for(lexer, offset, "<!--", false);
+}
+
+bool lexer_peek_for_html_comment_end(const lexer_T* lexer, const int offset) {
+  return lexer_peek_for(lexer, offset, "-->", false);
+}
+
+bool lexer_peek_erb_close_tag(const lexer_T* lexer, const int offset) {
+  return lexer_peek_for(lexer, offset, "%>", false);
+}
+
+bool lexer_peek_erb_dash_close_tag(const lexer_T* lexer, const int offset) {
+  return lexer_peek_for(lexer, offset, "-%>", false);
+}
+
+bool lexer_peek_erb_percent_close_tag(const lexer_T* lexer, const int offset) {
+  return lexer_peek_for(lexer, offset, "%%>", false);
+}
+
+bool lexer_peek_erb_end(const lexer_T* lexer, const int offset) {
+  return (
+    lexer_peek_erb_close_tag(lexer, offset) || lexer_peek_erb_dash_close_tag(lexer, offset)
+    || lexer_peek_erb_percent_close_tag(lexer, offset)
+  );
+}

data/src/location.c

@@ -0,0 +1,41 @@
+#include "include/location.h"
+#include "include/memory.h"
+#include "include/position.h"
+
+size_t location_sizeof(void) {
+  return sizeof(location_T);
+}
+
+location_T* location_init(position_T* start, position_T* end) {
+  location_T* location = safe_malloc(location_sizeof());
+
+  location->start = start;
+  location->end = end;
+
+  return location;
+}
+
+location_T* location_from(size_t start_line, size_t start_column, size_t end_line, size_t end_column) {
+  return location_init(position_init(start_line, start_column), position_init(end_line, end_column));
+}
+
+position_T* location_start(location_T* location) {
+  return location->start;
+}
+
+position_T* location_end(location_T* location) {
+  return location->end;
+}
+
+location_T* location_copy(location_T* location) {
+  if (location == NULL) { return NULL; }
+
+  return location_init(position_copy(location->start), position_copy(location->end));
+}
+
+void location_free(location_T* location) {
+  if (location->start != NULL) { position_free(location->start); }
+  if (location->end != NULL) { position_free(location->end); }
+
+  free(location);
+}

data/src/main.c

@@ -0,0 +1,162 @@
+#define _POSIX_C_SOURCE 199309L // Enables `clock_gettime()`
+
+#include "include/analyze.h"
+#include "include/ast_node.h"
+#include "include/ast_nodes.h"
+#include "include/ast_pretty_print.h"
+#include "include/buffer.h"
+#include "include/extract.h"
+#include "include/herb.h"
+#include "include/io.h"
+#include "include/ruby_parser.h"
+
+#include <stdio.h>
+#include <string.h>
+#include <time.h>
+
+void print_time_diff(const struct timespec start, const struct timespec end, const char* verb) {
+  const double seconds = (double) end.tv_sec - (double) start.tv_sec;
+  const double nanoseconds = (double) end.tv_nsec - (double) start.tv_nsec;
+  const double total_ns = seconds * 1e9 + nanoseconds;
+
+  const double us = total_ns / 1e3;
+  const double ms = total_ns / 1e6;
+  const double s = total_ns / 1e9;
+
+  printf("Finished");
+  printf(" %s ", verb);
+  printf("in:\n\n");
+
+  printf("  %8.0f µs\n", us);
+  printf("  %8.3f ms\n", ms);
+  printf("  %8.6f  s\n\n", s);
+}
+
+int main(const int argc, char* argv[]) {
+  if (argc < 2) {
+    printf("./herb [command] [options]\n\n");
+
+    printf("Herb 🌿 Powerful and seamless HTML-aware ERB parsing.\n\n");
+
+    printf("./herb lex [file]      -  Lex a file\n");
+    printf("./herb lex_json [file] -  Lex a file and return the result as json.\n");
+    printf("./herb parse [file]    -  Parse a file\n");
+    printf("./herb ruby [file]     -  Extract Ruby from a file\n");
+    printf("./herb html [file]     -  Extract HTML from a file\n");
+    printf("./herb prism [file]    -  Extract Ruby from a file and parse the Ruby source with Prism\n");
+
+    return 1;
+  }
+
+  if (argc < 3) {
+    printf("Please specify input file.\n");
+    return 1;
+  }
+
+  buffer_T output;
+
+  if (!buffer_init(&output)) { return 1; }
+
+  char* source = herb_read_file(argv[2]);
+
+  struct timespec start, end;
+  clock_gettime(CLOCK_MONOTONIC, &start);
+
+  if (strcmp(argv[1], "visit") == 0) {
+    AST_DOCUMENT_NODE_T* root = herb_parse(source);
+    clock_gettime(CLOCK_MONOTONIC, &end);
+
+    herb_analyze_parse_tree(root, source);
+
+    ast_pretty_print_node((AST_NODE_T*) root, 0, 0, &output);
+    printf("%s\n", output.value);
+
+    print_time_diff(start, end, "visiting");
+
+    ast_node_free((AST_NODE_T*) root);
+    buffer_free(&output);
+    free(source);
+
+    return 0;
+  }
+
+  if (strcmp(argv[1], "lex") == 0) {
+    herb_lex_to_buffer(source, &output);
+    clock_gettime(CLOCK_MONOTONIC, &end);
+
+    printf("%s\n", output.value);
+    print_time_diff(start, end, "lexing");
+
+    buffer_free(&output);
+    free(source);
+
+    return 0;
+  }
+
+  if (strcmp(argv[1], "lex_json") == 0) {
+    herb_lex_json_to_buffer(source, &output);
+
+    printf("%s\n", output.value);
+
+    buffer_free(&output);
+    free(source);
+
+    return 0;
+  }
+
+  if (strcmp(argv[1], "parse") == 0) {
+    AST_DOCUMENT_NODE_T* root = herb_parse(source);
+    clock_gettime(CLOCK_MONOTONIC, &end);
+
+    ast_pretty_print_node((AST_NODE_T*) root, 0, 0, &output);
+    printf("%s\n", output.value);
+
+    print_time_diff(start, end, "parsing");
+
+    ast_node_free((AST_NODE_T*) root);
+    buffer_free(&output);
+    free(source);
+
+    return 0;
+  }
+
+  if (strcmp(argv[1], "ruby") == 0) {
+    herb_extract_ruby_to_buffer(source, &output);
+    clock_gettime(CLOCK_MONOTONIC, &end);
+
+    printf("%s\n", output.value);
+    print_time_diff(start, end, "extracting Ruby");
+
+    buffer_free(&output);
+    free(source);
+
+    return 0;
+  }
+
+  if (strcmp(argv[1], "html") == 0) {
+    herb_extract_html_to_buffer(source, &output);
+    clock_gettime(CLOCK_MONOTONIC, &end);
+
+    printf("%s\n", output.value);
+    print_time_diff(start, end, "extracting HTML");
+
+    buffer_free(&output);
+    free(source);
+
+    return 0;
+  }
+
+  if (strcmp(argv[1], "prism") == 0) {
+    printf("HTML+ERB File: \n%s\n", source);
+
+    char* ruby_source = herb_extract(source, HERB_EXTRACT_LANGUAGE_RUBY);
+    printf("Extracted Ruby: \n%s\n", ruby_source);
+
+    herb_parse_ruby_to_stdout(ruby_source);
+
+    return 0;
+  }
+
+  printf("Unknown Command: %s\n", argv[1]);
+  return 1;
+}

data/src/memory.c

@@ -0,0 +1,53 @@
+#include "memory.h"
+
+#include <stdbool.h>
+#include <stdio.h>
+#include <stdlib.h>
+
+static void* safe_malloc_internal(const size_t size, const bool fail_fast) {
+  if (size == 0) { return NULL; }
+
+  void* pointer = malloc(size);
+
+  if (!pointer) {
+    fprintf(stderr, "Error: Failed to allocate %zu bytes.\n", size);
+    fflush(stderr);
+    if (fail_fast) { exit(1); }
+    return NULL;
+  }
+
+  return pointer;
+}
+
+static void* safe_realloc_internal(void* pointer, const size_t new_size, const bool fail_fast) {
+  if (new_size == 0) { return NULL; }
+
+  if (!pointer) { return safe_malloc_internal(new_size, fail_fast); }
+
+  void* new_pointer = realloc(pointer, new_size);
+
+  if (!new_pointer) {
+    fprintf(stderr, "Error: Memory reallocation failed (size: %zu bytes).\n", new_size);
+    fflush(stderr);
+    if (fail_fast) { exit(1); }
+    return NULL;
+  }
+
+  return new_pointer;
+}
+
+void* safe_malloc(const size_t size) {
+  return safe_malloc_internal(size, true);
+}
+
+void* nullable_safe_malloc(const size_t size) {
+  return safe_malloc_internal(size, false);
+}
+
+void* safe_realloc(void* pointer, const size_t new_size) {
+  return safe_realloc_internal(pointer, new_size, true);
+}
+
+void* nullable_safe_realloc(void* pointer, const size_t new_size) {
+  return safe_realloc_internal(pointer, new_size, false);
+}