minihtml 0.1.0

data/ext/minihtml_token_stream/minihtml_token_stream.c

@@ -0,0 +1,191 @@
+#include "ruby.h"
+#include "ruby/encoding.h"
+#include <stdint.h>
+#include <string.h>
+
+#define DEFINE_REUSABLE_SYMBOL(name) static ID id_type_##name; static VALUE sym_##name;
+#define INITIALIZE_REUSABLE_SYMBOL(name) id_type_##name = rb_intern(#name); sym_##name = ID2SYM(id_type_##name);
+
+DEFINE_REUSABLE_SYMBOL(whitespace);
+DEFINE_REUSABLE_SYMBOL(eof);
+DEFINE_REUSABLE_SYMBOL(kind);
+
+typedef struct {
+    VALUE tokens;
+    int tokens_idx;
+    long tokens_len;
+    VALUE look[2];
+    int marks[128];
+    int marks_idx;
+} stream_t;
+
+static void stream_free(void *ptr) {
+    xfree(ptr);
+}
+
+static size_t stream_memsize(const void *ptr) {
+    return sizeof(stream_t);
+}
+
+static void stream_mark(void *ptr) {
+    const stream_t *s = ptr;
+    if (s->tokens) rb_gc_mark(s->tokens);
+}
+
+static const rb_data_type_t stream_type = {
+    "MiniHTML::CSS::TokenStream",
+    {stream_mark, stream_free, stream_memsize,},
+    0, 0, RUBY_TYPED_FREE_IMMEDIATELY
+};
+
+static VALUE stream_alloc(const VALUE klass) {
+    stream_t *s = ALLOC(stream_t);
+    memset(s, 0, sizeof(stream_t));
+    return TypedData_Wrap_Struct(klass, &stream_type, s);
+}
+
+static VALUE stream_initialize(const VALUE self, VALUE tokens) {
+    Check_Type(tokens, T_ARRAY);
+    stream_t *s;
+    TypedData_Get_Struct(self, stream_t, &stream_type, s);
+    s->tokens = tokens;
+    s->tokens_idx = 0;
+    s->marks_idx = 0;
+    s->tokens_len = RARRAY_LEN(tokens);
+
+    // prime lookahead
+    s->look[0] = s->tokens_len >= 1 ? rb_ary_entry(tokens, 0) : Qnil;
+    s->look[1] = s->tokens_len >= 2 ? rb_ary_entry(tokens, 1) : Qnil;
+    return self;
+}
+
+static void rotate(stream_t *s) {
+    s->look[0] = s->look[1];
+    s->look[1] = s->tokens_idx + 1 < s->tokens_len ? rb_ary_entry(s->tokens, s->tokens_idx + 1) : Qnil;
+}
+
+#define UNWRAP_STREAM stream_t *s; TypedData_Get_Struct(self, stream_t, &stream_type, s);
+
+static VALUE stream_peek(const VALUE self) {
+    UNWRAP_STREAM;
+    return s->look[0];
+}
+
+static VALUE stream_peek1(const VALUE self) {
+    UNWRAP_STREAM;
+    return s->look[1];
+}
+
+static void stream_consume_c(stream_t *s) {
+    if (s->tokens_idx < s->tokens_len && s->tokens_idx + 1 < s->tokens_len) {
+        s->tokens_idx++;
+    }
+    rotate(s);
+}
+
+static VALUE stream_consume(const VALUE self) {
+    UNWRAP_STREAM;
+    const VALUE peek = s->look[0];
+    stream_consume_c(s);
+    return peek;
+}
+
+static VALUE stream_discard(const VALUE self) {
+    UNWRAP_STREAM;
+    stream_consume_c(s);
+    return Qnil;
+}
+
+static VALUE stream_create_mark(const VALUE self) {
+    UNWRAP_STREAM;
+    if (s->marks_idx >= 127) {
+        rb_raise(rb_eRuntimeError, "Too many marks in stream");
+    }
+
+    s->marks[s->marks_idx++] = s->tokens_idx;
+    return Qnil;
+}
+
+static VALUE stream_mark_restore(const VALUE self) {
+    UNWRAP_STREAM;
+    if (s->marks_idx == 0) {
+        rb_raise(rb_eRuntimeError, "BUG: No mark to restore");
+    }
+    const int mark = s->marks[s->marks_idx - 1];
+    s->tokens_idx = mark;
+    s->look[0] = s->tokens_len > mark ? rb_ary_entry(s->tokens, mark) : Qnil;
+    s->look[1] = s->tokens_len > mark + 1 ? rb_ary_entry(s->tokens, mark + 1) : Qnil;
+    s->marks_idx--;
+    return Qnil;
+}
+
+static VALUE stream_mark_pop(const VALUE self) {
+    UNWRAP_STREAM;
+    if (s->marks_idx == 0) {
+        rb_raise(rb_eRuntimeError, "BUG: No mark to pop");
+    }
+    s->marks_idx--;
+    return Qnil;
+}
+
+static inline bool is_eof(const VALUE v) { return v == Qnil; }
+
+static VALUE stream_is_empty(const VALUE self) {
+    UNWRAP_STREAM;
+    if (s->tokens_idx >= s->tokens_len || is_eof(s->look[0]))
+        return Qtrue;
+    return Qfalse;
+}
+
+static VALUE stream_peek_kind(const VALUE self) {
+    UNWRAP_STREAM;
+    if (s->look[0] == Qnil) return Qnil;
+    const VALUE k = rb_hash_aref(s->look[0], sym_kind);
+    Check_Type(k, T_SYMBOL);
+    return k;
+}
+
+static VALUE stream_peek_kind1(const VALUE self) {
+    UNWRAP_STREAM;
+    if (s->look[1] == Qnil) return Qnil;
+    const VALUE k = rb_hash_aref(s->look[1], sym_kind);
+    Check_Type(k, T_SYMBOL);
+    return k;
+}
+
+static VALUE stream_status(const VALUE self) {
+    UNWRAP_STREAM;
+    const VALUE h = rb_hash_new();
+    rb_hash_aset(h, ID2SYM(rb_intern("eof")), Qnil);
+    rb_hash_aset(h, ID2SYM(rb_intern("tokens")), s->tokens);
+    rb_hash_aset(h, ID2SYM(rb_intern("tokens_idx")), INT2NUM(s->tokens_idx));
+    rb_hash_aset(h, ID2SYM(rb_intern("tokens_len")), INT2NUM((int)s->tokens_len));
+    rb_hash_aset(h, ID2SYM(rb_intern("look0")), s->look[0]);
+    rb_hash_aset(h, ID2SYM(rb_intern("look1")), s->look[1]);
+    rb_hash_aset(h, ID2SYM(rb_intern("marks_idx")), INT2NUM(s->marks_idx));
+    return h;
+}
+
+RUBY_FUNC_EXPORTED void Init_minihtml_token_stream(void) {
+    const VALUE mMiniHTML = rb_define_module("MiniHTML");
+    const VALUE cStream = rb_define_class_under(mMiniHTML, "TokenStream", rb_cObject);
+
+    INITIALIZE_REUSABLE_SYMBOL(whitespace);
+    INITIALIZE_REUSABLE_SYMBOL(eof);
+    INITIALIZE_REUSABLE_SYMBOL(kind);
+
+    rb_define_alloc_func(cStream, stream_alloc);
+    rb_define_method(cStream, "initialize", stream_initialize, 1);
+    rb_define_method(cStream, "peek", stream_peek, 0);
+    rb_define_method(cStream, "peek1", stream_peek1, 0);
+    rb_define_method(cStream, "peek_kind", stream_peek_kind, 0);
+    rb_define_method(cStream, "peek_kind1", stream_peek_kind1, 0);
+    rb_define_method(cStream, "consume", stream_consume, 0);
+    rb_define_method(cStream, "discard", stream_discard, 0);
+
+    rb_define_method(cStream, "mark", stream_create_mark, 0);
+    rb_define_method(cStream, "restore", stream_mark_restore, 0);
+    rb_define_method(cStream, "pop", stream_mark_pop, 0);
+    rb_define_method(cStream, "empty?", stream_is_empty, 0);
+    rb_define_method(cStream, "status", stream_status, 0);
+}

data/lib/minihtml/ast/attr.rb

@@ -0,0 +1,21 @@
+# frozen_string_literal: true
+
+module MiniHTML
+  module AST
+    class Attr < Base
+      attr_accessor :name
+      attr_reader :value
+
+      def initialize(token)
+        super
+        @name = token[:literal]
+        @value = nil
+      end
+
+      def value=(token)
+        @position_end = token.position_end
+        @value = token
+      end
+    end
+  end
+end

data/lib/minihtml/ast/base.rb

@@ -0,0 +1,15 @@
+# frozen_string_literal: true
+
+module MiniHTML
+  module AST
+    class Base
+      attr_reader :position_start, :position_end, :original_token
+
+      def initialize(token)
+        @original_token = token
+        @position_start = Position.new(line: token[:start_line], column: token[:start_column], offset: token[:start_offset])
+        @position_end = Position.new(line: token[:end_line], column: token[:end_column], offset: token[:end_offset])
+      end
+    end
+  end
+end

data/lib/minihtml/ast/comment.rb

@@ -0,0 +1,14 @@
+# frozen_string_literal: true
+
+module MiniHTML
+  module AST
+    class Comment < Base
+      attr_accessor :literal
+
+      def initialize(token)
+        super
+        @literal = token[:literal]
+      end
+    end
+  end
+end

data/lib/minihtml/ast/executable.rb

@@ -0,0 +1,14 @@
+# frozen_string_literal: true
+
+module MiniHTML
+  module AST
+    class Executable < Base
+      attr_accessor :source
+
+      def initialize(token)
+        super
+        @source = token[:literal]
+      end
+    end
+  end
+end

data/lib/minihtml/ast/interpolation.rb

@@ -0,0 +1,14 @@
+# frozen_string_literal: true
+
+module MiniHTML
+  module AST
+    class Interpolation < Base
+      attr_accessor :values
+
+      def initialize(token)
+        super
+        @values = [AST::String.new(token)]
+      end
+    end
+  end
+end

data/lib/minihtml/ast/literal.rb

@@ -0,0 +1,14 @@
+# frozen_string_literal: true
+
+module MiniHTML
+  module AST
+    class Literal < Base
+      attr_accessor :value
+
+      def initialize(token)
+        super
+        @value = token[:literal]
+      end
+    end
+  end
+end

data/lib/minihtml/ast/plain_text.rb

@@ -0,0 +1,14 @@
+# frozen_string_literal: true
+
+module MiniHTML
+  module AST
+    class PlainText < Base
+      attr_accessor :literal
+
+      def initialize(token)
+        super
+        @literal = token[:literal]
+      end
+    end
+  end
+end

data/lib/minihtml/ast/position.rb

@@ -0,0 +1,15 @@
+# frozen_string_literal: true
+
+module MiniHTML
+  module AST
+    class Position
+      attr_reader :line, :column, :offset
+
+      def initialize(line:, column:, offset:)
+        @line = line
+        @column = column
+        @offset = offset
+      end
+    end
+  end
+end

data/lib/minihtml/ast/string.rb

@@ -0,0 +1,15 @@
+# frozen_string_literal: true
+
+module MiniHTML
+  module AST
+    class String < Base
+      attr_accessor :quote, :literal
+
+      def initialize(token)
+        super
+        @literal = token[:literal].gsub(/\\#{token[:quote_char]}/, token[:quote_char])
+        @quote = token[:quote_char]
+      end
+    end
+  end
+end

data/lib/minihtml/ast/tag.rb

@@ -0,0 +1,26 @@
+# frozen_string_literal: true
+
+module MiniHTML
+  module AST
+    class Tag < Base
+      attr_accessor :name, :self_closing, :attributes, :children, :bad_tag
+      alias self_closing? self_closing
+      alias bad_tag? bad_tag
+
+      def initialize(token)
+        super
+        lit = token[:literal]
+        if lit.start_with? "</"
+          @bad_tag = true
+          @name = token[:literal][2...]
+        else
+          @name = token[:literal][1...]
+        end
+
+        @self_closing = false
+        @attributes = []
+        @children = []
+      end
+    end
+  end
+end

data/lib/minihtml/ast.rb

@@ -0,0 +1,17 @@
+# frozen_string_literal: true
+
+require_relative "ast/position"
+require_relative "ast/base"
+require_relative "ast/attr"
+require_relative "ast/tag"
+require_relative "ast/plain_text"
+require_relative "ast/literal"
+require_relative "ast/string"
+require_relative "ast/executable"
+require_relative "ast/interpolation"
+require_relative "ast/comment"
+
+module MiniHTML
+  module AST
+  end
+end

data/lib/minihtml/parser.rb

@@ -0,0 +1,113 @@
+# frozen_string_literal: true
+
+module MiniHTML
+  class Parser
+    attr_reader :stream
+
+    def initialize(source)
+      scanner = MiniHTML::Scanner.new(source)
+      tokens = scanner.tokenize
+      raise ParseError.new(*scanner.errors) unless scanner.errors.empty?
+
+      @stream = MiniHTML::TokenStream.new(tokens)
+      @tokens = []
+    end
+
+    def parse
+      @tokens << parse_one until stream.empty?
+      @tokens
+    end
+
+    def parse_one
+      case stream.peek_kind
+      when :literal
+        AST::PlainText.new(stream.consume)
+      when :tag_begin
+        if stream.peek[:literal] == "<!--"
+          parse_comment
+        else
+          parse_tag
+        end
+      when :attr_value_literal
+        AST::Literal.new(stream.consume)
+      when :string
+        AST::String.new(stream.consume)
+      when :executable
+        AST::Executable.new(stream.consume)
+      when :string_interpolation
+        parse_string_interpolation
+      when :tag_closing_start
+        tag = AST::Tag.new(stream.consume)
+        discard_until_tag_end
+        tag
+      else
+        raise "Unexpected token type #{stream.peek_kind} on #parse_one"
+      end
+    end
+
+    def parse_comment
+      stream.consume
+      AST::Comment.new(stream.consume) unless stream.empty?
+    end
+
+    def parse_string_interpolation
+      interp = AST::Interpolation.new(stream.consume)
+      until stream.empty?
+        case stream.peek_kind
+        when :executable
+          interp.values << parse_one
+        when :string_interpolation
+          interp.values << AST::String.new(stream.consume)
+        when :string
+          interp.values << AST::String.new(stream.consume)
+          return interp
+        when :interpolated_executable
+          interp.values << AST::Executable.new(stream.consume)
+        else
+          raise "Unexpected token type #{stream.peek_kind} on #parse_string_interpolation"
+        end
+      end
+      interp
+    end
+
+    def discard_until_tag_end
+      stream.consume until stream.empty? || %i[tag_end tag_closing_end].include?(stream.peek_kind)
+      stream.consume # Consume tag_end or tag_closing_end
+    end
+
+    def parse_tag
+      tag = AST::Tag.new(stream.consume)
+      until stream.empty?
+        case stream.peek_kind
+        when :right_angled
+          stream.consume
+          # This tag has children...
+          tag.children << parse_one until stream.peek_kind == :tag_closing_start || stream.empty?
+        when :tag_closing_start
+          if stream.peek[:literal][2...] == tag.name
+            # Consume everything until a closing_end
+            discard_until_tag_end
+            return tag
+          end
+        when :tag_end
+          stream.consume
+          tag.self_closing = true
+          return tag
+        when :attr_key
+          tag.attributes << parse_attr
+        else
+          raise "Unexpected token type #{stream.peek_kind} on #parse_tag"
+        end
+      end
+    end
+
+    def parse_attr
+      att = AST::Attr.new(stream.consume)
+      return att unless stream.peek_kind == :equal
+
+      stream.consume # equal
+      att.value = parse_one
+      att
+    end
+  end
+end

data/lib/minihtml/version.rb

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+module MiniHTML
+  VERSION = "0.1.0"
+end

data/lib/minihtml.rb

@@ -0,0 +1,21 @@
+# frozen_string_literal: true
+
+require_relative "minihtml/version"
+require_relative "minihtml/minihtml_scanner"
+require_relative "minihtml/minihtml_token_stream"
+
+require_relative "minihtml/ast"
+require_relative "minihtml/parser"
+
+module MiniHTML
+  class Error < StandardError; end
+
+  class ParseError < Error
+    attr_reader :errors
+
+    def initialize(*errors)
+      @errors = errors
+      super("Parsing failed with #{errors.length} error#{"s" if errors.length > 1}: #{errors.join(", ")}")
+    end
+  end
+end

metadata

@@ -0,0 +1,74 @@
+--- !ruby/object:Gem::Specification
+name: minihtml
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- Vito Sartori
+bindir: exe
+cert_chain: []
+date: 1980-01-02 00:00:00.000000000 Z
+dependencies: []
+description: MiniHTML is a small HTML parser intended for React-like HTML-JSX syntax
+email:
+- hey@vito.io
+executables: []
+extensions:
+- ext/minihtml_scanner/extconf.rb
+- ext/minihtml_token_stream/extconf.rb
+extra_rdoc_files: []
+files:
+- ".editorconfig"
+- ".rspec"
+- ".rubocop.yml"
+- ".ruby-version"
+- CODE_OF_CONDUCT.md
+- LICENSE
+- README.md
+- Rakefile
+- ext/minihtml_scanner/extconf.rb
+- ext/minihtml_scanner/minihtml_scanner.c
+- ext/minihtml_scanner/minihtml_scanner.h
+- ext/minihtml_token_stream/extconf.rb
+- ext/minihtml_token_stream/minihtml_token_stream.c
+- lib/minihtml.rb
+- lib/minihtml/ast.rb
+- lib/minihtml/ast/attr.rb
+- lib/minihtml/ast/base.rb
+- lib/minihtml/ast/comment.rb
+- lib/minihtml/ast/executable.rb
+- lib/minihtml/ast/interpolation.rb
+- lib/minihtml/ast/literal.rb
+- lib/minihtml/ast/plain_text.rb
+- lib/minihtml/ast/position.rb
+- lib/minihtml/ast/string.rb
+- lib/minihtml/ast/tag.rb
+- lib/minihtml/parser.rb
+- lib/minihtml/version.rb
+homepage: https://github.com/heyvito/minihtml
+licenses:
+- MIT
+metadata:
+  allowed_push_host: https://rubygems.org
+  homepage_uri: https://github.com/heyvito/minihtml
+  source_code_uri: https://github.com/heyvito/minihtml
+  changelog_uri: https://github.com/heyvito/minihtml
+  rubygems_mfa_required: 'true'
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '3.4'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 3.6.9
+specification_version: 4
+summary: MiniHTML is a small HTML parser intended for React-like HTML-JSX syntax
+test_files: []