ruxml 0.0.1

data/ext/ruxml/parser.hpp

@@ -0,0 +1,140 @@
+#pragma once
+
+#include <cstdint>
+
+#include "str.hpp"
+
+#define TOKEN2(a) (TokenType)(((uint16_t)((a)[1])<<7)+(uint16_t)((a)[0]))
+
+enum TokenType : uint16_t {
+  TOK_INVALID = 0,
+  TOK_L_ANGLED = '<',
+  TOK_R_ANGLED = '>',
+  TOK_EQUALS = '=',
+  TOK_COLON = ':',
+  TOK_SLASH = '/',
+  TOK_QUESTION = '?',
+  TOK_BANG = '!',
+  TOK_HYPHEN = '-',
+  TOK_ONE_CHAR = 127,
+
+  TOK_TAG_START_CLOSE = TOKEN2("</"),
+  TOK_TAG_XML_START = TOKEN2("<?"),
+  TOK_TAG_XML_END = TOKEN2("?>"),
+  TOK_TAG_SELF_CLOSE = TOKEN2("/>"),
+  TOK_TWO_CHAR = 128 * 127,
+
+  TOK_IDENTIFIER = 20000,
+  TOK_VALUE,
+  TOK_TEXT,
+  TOK_COMMENT_START,
+  TOK_COMMENT_END,
+};
+
+struct Token {
+  TokenType type;
+  int64_t line;
+  int64_t c0;
+  int64_t c1;
+  int64_t offset;
+  String text;
+};
+
+enum LexerAction : uint8_t {
+  LA_INVALID,
+  LA_ONE_CHAR,
+  LA_TWO_CHAR_END,
+  LA_WHITESPACE,
+  LA_NEWLINE,
+  LA_IDENTIFIER,
+  LA_VALUE
+};
+
+enum LexerMode : uint8_t {
+  LM_OUT,
+  LM_TAG,
+  LM_COMMENT
+};
+
+enum ParserSourceType {
+  PST_NONE = 0,
+  PST_MEMORY,
+  PST_MMAP
+};
+
+enum NodeType {
+  NODE_INVALID,
+  NODE_ELEMENT_BEGIN,
+  NODE_ELEMENT_END,
+  NODE_TEXT,
+  NODE_XML_HEADER,
+  NODE_COMMENT,
+
+  MAX_NODE_TYPES
+};
+
+struct Node {
+  NodeType type;
+  int64_t line;
+  int64_t c0;
+  int64_t c1;
+  int64_t offset;
+  int64_t depth;
+
+  bool self_closing;
+  String text;
+};
+
+struct Parser {
+  String source;
+  ParserSourceType source_type;
+  char *buffer;
+  int64_t length;
+
+  char *ptr;
+  char *end_ptr;
+
+  int64_t line;
+  int64_t col;
+
+  bool done;
+  bool errored;
+  LexerMode mode;
+
+  LexerAction tag_initial_map[256];
+
+  uint8_t identifier_map[256];
+
+  Token token;
+  Node node;
+
+  int64_t depth;
+};
+
+void parser_init(Parser *parser);
+bool parser_open_memory(Parser *parser, String name, const char *memory, int64_t offset = 0, int64_t length = 0);
+bool parser_open_file_mmap(Parser *parser, String filename, int64_t offset = 0, int64_t length = 0);
+void parser_destroy(Parser *parser);
+
+void print_error_start(Parser *parser, Token token);
+bool expect_type(Parser *parser, TokenType type);
+
+Token read_token(Parser *parser); // Internal only: use get_token instead
+Node read_node(Parser *parser); // Internal only: use get_node instead
+
+inline Token peek_token(Parser *parser) { return parser->token; }
+
+inline Token get_token(Parser *parser) {
+  parser->token = read_token(parser);
+  if (!parser->token.type) parser->done = true;
+  return parser->token;
+}
+
+inline Node get_node(Parser *parser) {
+  parser->node = read_node(parser);
+  return parser->node;
+}
+
+void print_token(Token token);
+
+void print_node(Node node);

data/ext/ruxml/ruxml.cpp

@@ -0,0 +1,273 @@
+#include "parser.hpp"
+#include <ruby/ruby.h>
+
+extern "C"
+{
+
+VALUE ruxmlModule;
+VALUE ruxmlParser;
+VALUE ruxmlNode;
+
+ID node_type_ids[MAX_NODE_TYPES];
+
+//
+// Helpers
+//
+
+VALUE rbstr_from_str(String str) { return rb_str_export_locale(rb_str_new(str.data, str.length)); }
+
+String str_from_rbstr(VALUE rbstr) { return String{(int) RSTRING_LEN(rbstr), StringValuePtr(rbstr)}; }
+
+//
+// Node
+//
+
+static Node *Node_instance(VALUE self) {
+  return (Node *) RDATA(self)->data;
+}
+
+static size_t Node_size(const void *data) {
+  return sizeof(Node);
+}
+
+static void Node_free(void *data) {
+  free(data);
+}
+
+rb_data_type_t Node_data_type = {
+    "Node",
+    {NULL, Node_free, Node_size},
+    0, 0,
+    RUBY_TYPED_FREE_IMMEDIATELY
+};
+
+static VALUE Node_allocate(VALUE self) {
+  Node *node;
+  return TypedData_Make_Struct(self, Node, &Node_data_type, node);
+}
+
+static VALUE Node_initialize(VALUE self) {
+  Node *node;
+  TypedData_Get_Struct(self, Node, &Node_data_type, node);
+  *node = Node{};
+  return self;
+}
+
+static VALUE Node_column_start(VALUE self) {
+  auto node = Node_instance(self);
+  return INT2NUM(node->c0);
+}
+
+static VALUE Node_line(VALUE self) {
+  auto node = Node_instance(self);
+  return INT2NUM(node->line);
+}
+
+static VALUE Node_offset(VALUE self) {
+  auto node = Node_instance(self);
+  return INT2NUM(node->offset);
+}
+
+static VALUE Node_text(VALUE self) {
+  auto node = Node_instance(self);
+  return rbstr_from_str(node->text);
+}
+
+static VALUE Node_type(VALUE self) {
+  auto node = Node_instance(self);
+  return ID2SYM(node_type_ids[node->type]);
+}
+
+static VALUE Node_self_closing(VALUE self) {
+  auto node = Node_instance(self);
+  return node->self_closing ? Qtrue : Qfalse;
+}
+
+//
+// Parser
+//
+
+static Parser *Parser_instance(VALUE self) {
+  return (Parser *) RDATA(self)->data;
+}
+
+static size_t Parser_size(const void *data) {
+  return sizeof(Parser);
+}
+
+static void Parser_free(void *data) {
+  parser_destroy((Parser *) data);
+  free(data);
+}
+
+rb_data_type_t Parser_data_type = {
+    "Parser",
+    {NULL, Parser_free, Parser_size},
+    0, 0,
+    RUBY_TYPED_FREE_IMMEDIATELY
+};
+
+static VALUE Parser_allocate(VALUE self) {
+  Parser *parser;
+  return TypedData_Make_Struct(self, Parser, &Parser_data_type, parser);
+}
+
+static VALUE Parser_initialize(VALUE self) {
+  Parser *parser;
+  TypedData_Get_Struct(self, Parser, &Parser_data_type, parser);
+
+  *parser = Parser{};
+  parser_init(parser);
+  return self;
+}
+
+static VALUE Parser_open_string(int argc, VALUE* argv, VALUE self) {
+  VALUE name;
+  VALUE data;
+  VALUE offset;
+  VALUE length;
+  rb_scan_args(argc, argv, "22", &name, &data, &offset, &length);
+
+  Check_Type(name, T_STRING);
+  Check_Type(data, T_STRING);
+
+  int64_t data_offset = 0;
+  if (!NIL_P(offset)) {
+    Check_Type(offset, T_FIXNUM);
+    data_offset = NUM2INT(offset);
+  }
+
+  int64_t data_length;
+  if (NIL_P(length)) {
+    data_length = RSTRING_LEN(data);
+  } else {
+    Check_Type(length, T_FIXNUM);
+    data_length = NUM2INT(length);
+  }
+
+  auto parser = Parser_instance(self);
+  auto success = parser_open_memory(parser, str_from_rbstr(name), StringValuePtr(data), data_offset, data_length);
+  return success ? Qtrue : Qfalse;
+}
+
+static VALUE Parser_open_file(int argc, VALUE* argv, VALUE self) {
+  VALUE filename;
+  VALUE offset;
+  VALUE length;
+  rb_scan_args(argc, argv, "12", &filename, &offset, &length);
+
+  Check_Type(filename, T_STRING);
+
+  int64_t data_offset = 0;
+  if (!NIL_P(offset)) {
+    Check_Type(offset, T_FIXNUM);
+    data_offset = NUM2INT(offset);
+  }
+
+  int64_t data_length = 0;
+  if (!NIL_P(length)) {
+    Check_Type(length, T_FIXNUM);
+    data_length = NUM2INT(length);
+  }
+
+  auto parser = Parser_instance(self);
+  auto success = parser_open_file_mmap(parser, str_from_rbstr(filename), data_offset, data_length);
+  return success ? Qtrue : Qfalse;
+}
+
+static VALUE Parser_node(VALUE self) {
+  auto parser = Parser_instance(self);
+  auto node_ptr = raw_allocate_type(Node);
+  *node_ptr = parser->node;
+  return TypedData_Wrap_Struct(ruxmlNode, &Node_data_type, node_ptr);
+}
+
+static VALUE Parser_next_node(VALUE self) {
+  auto parser = Parser_instance(self);
+  get_node(parser);
+  return parser->done ? Qfalse : Qtrue;
+}
+
+static VALUE Parser_done(VALUE self) {
+  auto parser = Parser_instance(self);
+  return parser->done ? Qtrue : Qfalse;
+}
+
+static VALUE Parser_errored(VALUE self) {
+  auto parser = Parser_instance(self);
+  return parser->errored ? Qtrue : Qfalse;
+}
+
+static VALUE Parser_node_column_start(VALUE self) {
+  auto parser = Parser_instance(self);
+  return INT2NUM(parser->node.c0);
+}
+
+static VALUE Parser_node_line(VALUE self) {
+  auto parser = Parser_instance(self);
+  return INT2NUM(parser->node.line);
+}
+
+static VALUE Parser_node_offset(VALUE self) {
+  auto parser = Parser_instance(self);
+  return INT2NUM(parser->node.offset);
+}
+
+static VALUE Parser_node_text(VALUE self) {
+  auto parser = Parser_instance(self);
+  return rbstr_from_str(parser->node.text);
+}
+
+static VALUE Parser_node_type(VALUE self) {
+  auto parser = Parser_instance(self);
+  return ID2SYM(node_type_ids[parser->node.type]);
+}
+
+static VALUE Parser_node_self_closing(VALUE self) {
+  auto parser = Parser_instance(self);
+  return parser->node.self_closing ? Qtrue : Qfalse;
+}
+
+//
+// Init
+//
+
+void Init_ruxml() {
+  node_type_ids[NODE_INVALID] = rb_intern("invalid");
+  node_type_ids[NODE_ELEMENT_BEGIN] = rb_intern("begin");
+  node_type_ids[NODE_ELEMENT_END] = rb_intern("end");
+  node_type_ids[NODE_TEXT] = rb_intern("text");
+  node_type_ids[NODE_XML_HEADER] = rb_intern("xml_header");
+  node_type_ids[NODE_COMMENT] = rb_intern("comment");
+
+  ruxmlModule = rb_define_module("RUXML");
+
+  ruxmlNode = rb_define_class_under(ruxmlModule, "Node", rb_cData);
+  rb_define_alloc_func(ruxmlNode, Node_allocate);
+  rb_define_method(ruxmlNode, "initialize", reinterpret_cast<VALUE (*)(...)>(Node_initialize), 0);
+  rb_define_method(ruxmlNode, "column_start", reinterpret_cast<VALUE (*)(...)>(Node_column_start), 0);
+  rb_define_method(ruxmlNode, "line", reinterpret_cast<VALUE (*)(...)>(Node_line), 0);
+  rb_define_method(ruxmlNode, "offset", reinterpret_cast<VALUE (*)(...)>(Node_offset), 0);
+  rb_define_method(ruxmlNode, "text", reinterpret_cast<VALUE (*)(...)>(Node_text), 0);
+  rb_define_method(ruxmlNode, "type", reinterpret_cast<VALUE (*)(...)>(Node_type), 0);
+  rb_define_method(ruxmlNode, "self_closing", reinterpret_cast<VALUE (*)(...)>(Node_self_closing), 0);
+
+  ruxmlParser = rb_define_class_under(ruxmlModule, "Parser", rb_cData);
+  rb_define_alloc_func(ruxmlParser, Parser_allocate);
+  rb_define_method(ruxmlParser, "initialize", reinterpret_cast<VALUE (*)(...)>(Parser_initialize), 0);
+  rb_define_method(ruxmlParser, "open_string", reinterpret_cast<VALUE (*)(...)>(Parser_open_string), -1);
+  rb_define_method(ruxmlParser, "open_file", reinterpret_cast<VALUE (*)(...)>(Parser_open_file), -1);
+  rb_define_method(ruxmlParser, "node", reinterpret_cast<VALUE (*)(...)>(Parser_node), 0);
+  rb_define_method(ruxmlParser, "next_node", reinterpret_cast<VALUE (*)(...)>(Parser_next_node), 0);
+  rb_define_method(ruxmlParser, "done", reinterpret_cast<VALUE (*)(...)>(Parser_done), 0);
+  rb_define_method(ruxmlParser, "errored", reinterpret_cast<VALUE (*)(...)>(Parser_errored), 0);
+
+  rb_define_method(ruxmlParser, "node_column_start", reinterpret_cast<VALUE (*)(...)>(Parser_node_column_start), 0);
+  rb_define_method(ruxmlParser, "node_line", reinterpret_cast<VALUE (*)(...)>(Parser_node_line), 0);
+  rb_define_method(ruxmlParser, "node_offset", reinterpret_cast<VALUE (*)(...)>(Parser_node_offset), 0);
+  rb_define_method(ruxmlParser, "node_text", reinterpret_cast<VALUE (*)(...)>(Parser_node_text), 0);
+  rb_define_method(ruxmlParser, "node_type", reinterpret_cast<VALUE (*)(...)>(Parser_node_type), 0);
+  rb_define_method(ruxmlParser, "node_self_closing", reinterpret_cast<VALUE (*)(...)>(Parser_node_self_closing), 0);
+}
+
+}

data/ext/ruxml/str.cpp

@@ -0,0 +1,216 @@
+//
+// Created by divan on 27/12/18.
+//
+
+#include <cstdarg>
+#include <cstdio>
+#include "str.hpp"
+
+uint64_t zstr_length(const char *str) {
+  uint64_t result = 0;
+  while (*(str++)) result++;
+  return result;
+}
+
+char *zstr_dup(const char *str, int64_t size) {
+  char *result = raw_allocate_string_zt(size);
+  memcpy(result, str, sizeof(char) * size);
+  return result;
+}
+
+char *zstr_dup(const char *str) {
+  return zstr_dup(str, zstr_length(str));
+}
+
+char *str_to_zstr(Allocator *allocator, String s) {
+  char *result = (char *) allocate_size(allocator, (size_t) (s.length + 1));
+  memcpy(result, s.data, sizeof(char) * s.length);
+  result[s.length] = 0;
+  return result;
+}
+
+char *str_to_zstr(String s) { return str_to_zstr(temp_allocator, s); }
+
+String str_dup(Allocator *allocator, String s) {
+  String result;
+  result.length = s.length;
+  result.data = (char *) allocate_size(allocator, (size_t) (s.length + 1));
+  memcpy(result.data, s.data, sizeof(char) * s.length);
+  result.data[s.length] = 0;
+  return result;
+}
+
+String str_dup(String s) { return str_dup(temp_allocator, s); }
+
+String str_dup(Allocator *allocator, const char *str) {
+  return str_dup(allocator, str, static_cast<int>(zstr_length(str)));
+}
+
+String str_dup(const char *str) { return str_dup(temp_allocator, str); }
+
+String str_dup(Allocator *allocator, const char *str, int length) {
+  String result;
+  result.length = length;
+  result.data = allocate_zstring(allocator, length);
+  memcpy(result.data, str, sizeof(char) * length);
+  result.data[length] = 0;
+  return result;
+}
+
+String str_dup(const char *str, int length) { return str_dup(temp_allocator, str, length); }
+
+int zstr_find_last(const char *str, char c) {
+  int result = -1;
+  int index = 0;
+  while (*str != 0) {
+    if (*str == c) result = index;
+    str++;
+    index++;
+  }
+  return result;
+}
+
+int str_find_last(String s, char c, int after) {
+  int result = -1;
+  int index = 0;
+  char *str = s.data + after;
+  for (int i = after; i < s.length; i++) {
+    if (*str == c) result = index;
+    str++;
+    index++;
+  }
+  return result;
+}
+
+int str_find_first(String s, char c, int after) {
+  char *str = s.data + after;
+  for (int i = after; i < s.length; i++) {
+    if (*str == c) return i;
+    str++;
+  }
+  return -1;
+}
+
+bool str_equal(String a, String b) {
+  if (a.length != b.length) return false;
+  return memcmp(a.data, b.data, a.length) == 0;
+}
+
+bool str_equal(String a, const char *b) {
+  return str_equal(a, b, zstr_length(b));
+}
+
+bool str_equal(String a, const char *b_data, int b_length) {
+  if (a.length != b_length) return false;
+  return memcmp(a.data, b_data, a.length) == 0;
+}
+
+int str_compare(String a, String b) {
+  if (a.length < b.length) return -1;
+  if (b.length > a.length) return 1;
+  return memcmp(a.data, b.data, a.length);
+}
+
+bool str_empty(String s) {
+  return s.length == 0 || !s.data;
+}
+
+bool parse_int(String string, int32_t *result_ptr) {
+  bool valid = false;
+  int result = 0;
+
+  auto buffer = string.data;
+
+  for (int i = 0; i < string.length; i++) {
+    char c = buffer[0];
+    if (c >= '0' && c <= '9') {
+      result *= 10;
+      result += c - '0';
+      valid = true;
+    } else {
+      valid = false;
+      break;
+    }
+    buffer++;
+  }
+
+  *result_ptr = result;
+  return valid;
+}
+
+bool parse_int64(String string, int64_t *result_ptr) {
+  bool valid = false;
+  int result = 0;
+
+  auto buffer = string.data;
+
+  for (int i = 0; i < string.length; i++) {
+    char c = buffer[0];
+    if (c >= '0' && c <= '9') {
+      result *= 10;
+      result += c - '0';
+      valid = true;
+    } else {
+      valid = false;
+      break;
+    }
+    buffer++;
+  }
+
+  *result_ptr = result;
+  return valid;
+}
+
+String str_print(Allocator *allocator, const char *fmt, ...) {
+  char buffer[1024];
+
+  va_list v, v2;
+  va_start(v, fmt);
+  va_copy(v2, v);
+
+  auto res = vsnprintf(buffer, sizeof(buffer), fmt, v);
+  va_end(v);
+
+  if (res <= array_size(buffer)) {
+    return str_dup(allocator, buffer, res);
+  } else {
+    int big_size = res + 1;
+
+    String result;
+    result.length = res;
+    result.data = allocate_zstring(allocator, res);
+    res = vsnprintf(result.data, static_cast<size_t>(big_size), fmt, v2);
+
+    va_end(v2);
+    assert(res >= 0);
+
+    return result;
+  }
+}
+
+String str_print(const char *fmt, ...) {
+  char buffer[1024];
+
+  va_list v, v2;
+  va_start(v, fmt);
+  va_copy(v2, v);
+
+  auto res = vsnprintf(buffer, sizeof(buffer), fmt, v);
+  va_end(v);
+
+  if (res <= array_size(buffer)) {
+    return str_dup(temp_allocator, buffer, res);
+  } else {
+    int big_size = res + 1;
+
+    String result;
+    result.length = res;
+    result.data = allocate_zstring(temp_allocator, res);
+    res = vsnprintf(result.data, static_cast<size_t>(big_size), fmt, v2);
+
+    va_end(v2);
+    assert(res >= 0);
+
+    return result;
+  }
+}