minicss 0.1.6 → 0.1.7

data/ext/minicss_scanner/minicss_scanner.h

@@ -0,0 +1,132 @@
+//
+// Created by Vito Sartori on 11/10/25.
+//
+
+#ifndef MINICSS_SCANNER_MINICSS_SCANNER_H
+#define MINICSS_SCANNER_MINICSS_SCANNER_H
+
+#define EOF_CP   (-1)
+#define NEWLINE  0x0A
+#define CP(c)    ((int)(c))
+#define REVERSE_SOLIDUS '\\'
+#define QUOTATION_MARK '"'
+#define APOSTROPHE '\''
+#define NUMBER_SIGN '#'
+#define LEFT_PARENTHESIS '('
+#define RIGHT_PARENTHESIS ')'
+#define PLUS_SIGN '+'
+#define FULL_STOP '.'
+#define HYPHEN_MINUS '-'
+#define PERCENTAGE_SIGN '%'
+#define COMMA ','
+#define GREATER_THAN '>'
+#define COLON ':'
+#define SEMICOLON ';'
+#define LESS_THAN '<'
+#define EXCLAMATION_MARK '!'
+#define COMMERCIAL_AT '@'
+#define LEFT_SQUARE_BRACKET '['
+#define RIGHT_SQUARE_BRACKET ']'
+#define LEFT_CURLY '{'
+#define RIGHT_CURLY '}'
+
+#define DEFINE_REUSABLE_SYMBOL(name) static ID id_type_##name; static VALUE sym_##name;
+#define INITIALIZE_REUSABLE_SYMBOL(name) id_type_##name = rb_intern(#name); sym_##name = ID2SYM(id_type_##name);
+
+DEFINE_REUSABLE_SYMBOL(line);
+DEFINE_REUSABLE_SYMBOL(column);
+DEFINE_REUSABLE_SYMBOL(offset);
+DEFINE_REUSABLE_SYMBOL(whitespace);
+DEFINE_REUSABLE_SYMBOL(start);
+DEFINE_REUSABLE_SYMBOL(end);
+DEFINE_REUSABLE_SYMBOL(unicode_range);
+DEFINE_REUSABLE_SYMBOL(new);
+DEFINE_REUSABLE_SYMBOL(bad_string);
+DEFINE_REUSABLE_SYMBOL(literal);
+DEFINE_REUSABLE_SYMBOL(quoting);
+DEFINE_REUSABLE_SYMBOL(string);
+DEFINE_REUSABLE_SYMBOL(value);
+DEFINE_REUSABLE_SYMBOL(type);
+DEFINE_REUSABLE_SYMBOL(integer);
+DEFINE_REUSABLE_SYMBOL(number);
+DEFINE_REUSABLE_SYMBOL(sign_character);
+DEFINE_REUSABLE_SYMBOL(unit);
+DEFINE_REUSABLE_SYMBOL(dimension);
+DEFINE_REUSABLE_SYMBOL(percentage);
+DEFINE_REUSABLE_SYMBOL(function);
+DEFINE_REUSABLE_SYMBOL(name);
+DEFINE_REUSABLE_SYMBOL(ident);
+DEFINE_REUSABLE_SYMBOL(url);
+DEFINE_REUSABLE_SYMBOL(bad_url);
+DEFINE_REUSABLE_SYMBOL(flag);
+DEFINE_REUSABLE_SYMBOL(hash);
+DEFINE_REUSABLE_SYMBOL(delim);
+DEFINE_REUSABLE_SYMBOL(left_parenthesis);
+DEFINE_REUSABLE_SYMBOL(right_parenthesis);
+DEFINE_REUSABLE_SYMBOL(comma);
+DEFINE_REUSABLE_SYMBOL(cdc);
+DEFINE_REUSABLE_SYMBOL(colon);
+DEFINE_REUSABLE_SYMBOL(semicolon);
+DEFINE_REUSABLE_SYMBOL(cdo);
+DEFINE_REUSABLE_SYMBOL(at_keyword);
+DEFINE_REUSABLE_SYMBOL(left_square_bracket);
+DEFINE_REUSABLE_SYMBOL(right_square_bracket);
+DEFINE_REUSABLE_SYMBOL(left_curly);
+DEFINE_REUSABLE_SYMBOL(right_curly);
+
+// Definitions -----------------------------------------------------------------
+
+/**
+ * scanner_t - State for a CSS tokenizer scanner.
+ *
+ * Holds the current position, lookahead, and bookkeeping needed to
+ * tokenize a UTF-8 encoded CSS source string. This structure is the
+ * central state passed around by the tokenizer.
+ *
+ * Fields:
+ *   str   - Ruby String VALUE containing the source (kept alive
+ *           to ensure the underlying buffer remains valid).
+ *
+ *   p     - Current byte pointer into the UTF-8 source.
+ *   end   - Pointer to one past the last byte of the source.
+ *
+ *   idx_cp - Index of the current code point (0-based).
+ *   line   - Current line number (1-based, for error reporting).
+ *   col    - Current column number (1-based, for error reporting).
+ *
+ *   look  - Lookahead buffer of decoded code points:
+ *             look[0] = next code point (peek)
+ *             look[1] = second ahead
+ *             look[2] = third ahead
+ *             look[3] = fourth ahead
+ *           Used for disambiguating tokens (e.g. distinguishing
+ *           between `--ident` and `-->`).
+ *
+ * Notes:
+ *   - `p` is advanced as tokens are consumed.
+ *   - `line` and `col` are updated to provide CSS-spec-compliant
+ *     source location information.
+ *   - Lookahead avoids re-decoding UTF-8 sequences during parsing.
+ */
+typedef struct {
+    VALUE str;
+    VALUE owner;
+    VALUE tokens;
+    bool allow_unicode_ranges;
+    const uint8_t *p;
+    const uint8_t *end;
+    long idx_cp;
+    long line;
+    long col;
+    int look[4];
+    long start_token_offset;
+    long start_token_line;
+    long start_token_column;
+} scanner_t;
+
+// Forward definitions ---------------------------------------------------------
+static void scanner_consume_ident_like_token(scanner_t *sc);
+static void scanner_consume_url_token(scanner_t *sc);
+static void scanner_consume_bad_url(scanner_t *sc);
+
+#endif //MINICSS_SCANNER_MINICSS_SCANNER_H

data/ext/minicss_token_stream/extconf.rb

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+require "mkmf"
+
+create_makefile("minicss_token_stream")

data/ext/minicss_token_stream/minicss_token_stream.c

@@ -0,0 +1,195 @@
+#include "ruby.h"
+#include "ruby/encoding.h"
+#include <stdint.h>
+#include <string.h>
+
+#define DEFINE_REUSABLE_SYMBOL(name) static ID id_type_##name; static VALUE sym_##name;
+#define INITIALIZE_REUSABLE_SYMBOL(name) id_type_##name = rb_intern(#name); sym_##name = ID2SYM(id_type_##name);
+
+DEFINE_REUSABLE_SYMBOL(whitespace);
+DEFINE_REUSABLE_SYMBOL(eof);
+static VALUE cTokenClass;
+
+typedef struct {
+    VALUE eof;
+    VALUE tokens;
+    int tokens_idx;
+    long tokens_len;
+    VALUE look[2];
+    int marks[128];
+    int marks_idx;
+} stream_t;
+
+static void stream_free(void *ptr) {
+    xfree(ptr);
+}
+
+static size_t stream_memsize(const void *ptr) {
+    return sizeof(stream_t);
+}
+
+static void stream_mark(void *ptr) {
+    const stream_t *s = ptr;
+    if (s->tokens) rb_gc_mark(s->tokens);
+    if (s->eof) rb_gc_mark(s->eof);
+}
+
+static const rb_data_type_t stream_type = {
+    "MiniCSS::CSS::TokenStream",
+    {stream_mark, stream_free, stream_memsize,},
+    0, 0, RUBY_TYPED_FREE_IMMEDIATELY
+};
+
+static VALUE stream_alloc(const VALUE klass) {
+    stream_t *s = ALLOC(stream_t);
+    memset(s, 0, sizeof(stream_t));
+    return TypedData_Wrap_Struct(klass, &stream_type, s);
+}
+
+static VALUE stream_initialize(const VALUE self, VALUE tokens) {
+    Check_Type(tokens, T_ARRAY);
+    stream_t *s;
+    TypedData_Get_Struct(self, stream_t, &stream_type, s);
+    s->tokens = tokens;
+    s->tokens_idx = 0;
+    s->marks_idx = 0;
+    s->tokens_len = RARRAY_LEN(tokens);
+    s->eof = rb_const_get(cTokenClass, rb_intern("EOF"));
+
+    // prime lookahead
+    s->look[0] = s->tokens_len >= 1 ? rb_ary_entry(tokens, 0) : s->eof;
+    s->look[1] = s->tokens_len >= 2 ? rb_ary_entry(tokens, 1) : s->eof;
+    return self;
+}
+
+static void rotate(stream_t *s) {
+    s->look[0] = s->look[1];
+    s->look[1] = s->tokens_idx + 1 < s->tokens_len ? rb_ary_entry(s->tokens, s->tokens_idx + 1) : s->eof;
+}
+
+#define UNWRAP_STREAM stream_t *s; TypedData_Get_Struct(self, stream_t, &stream_type, s);
+
+static VALUE stream_peek(const VALUE self) {
+    UNWRAP_STREAM;
+    return s->look[0];
+}
+
+static void stream_consume_c(stream_t *s) {
+    if (s->tokens_idx < s->tokens_len && s->tokens_idx + 1 < s->tokens_len) {
+        s->tokens_idx++;
+    }
+    rotate(s);
+}
+
+static VALUE stream_consume(const VALUE self) {
+    UNWRAP_STREAM;
+    const VALUE peek = s->look[0];
+    stream_consume_c(s);
+    return peek;
+}
+
+static VALUE stream_discard(const VALUE self) {
+    UNWRAP_STREAM;
+    stream_consume_c(s);
+    return Qnil;
+}
+
+static VALUE stream_discard_whitespace(const VALUE self) {
+    UNWRAP_STREAM;
+    for (;;) {
+        const VALUE k = rb_funcall(s->look[0], rb_intern("kind"), 0);
+        Check_Type(k, T_SYMBOL);
+        if (rb_sym2id(k) != id_type_whitespace) break;
+        stream_consume_c(s);
+    }
+    return Qnil;
+}
+
+static VALUE stream_create_mark(const VALUE self) {
+    UNWRAP_STREAM;
+    if (s->marks_idx >= 127) {
+        rb_raise(rb_eRuntimeError, "Too many marks in stream");
+    }
+
+    s->marks[s->marks_idx++] = s->tokens_idx;
+    return Qnil;
+}
+
+static VALUE stream_mark_restore(const VALUE self) {
+    UNWRAP_STREAM;
+    if (s->marks_idx == 0) {
+        rb_raise(rb_eRuntimeError, "BUG: No mark to restore");
+    }
+    const int mark = s->marks[s->marks_idx - 1];
+    s->tokens_idx = mark;
+    s->look[0] = s->tokens_len > mark ? rb_ary_entry(s->tokens, mark) : s->eof;
+    s->look[1] = s->tokens_len > mark + 1 ? rb_ary_entry(s->tokens, mark + 1) : s->eof;
+    s->marks_idx--;
+    return Qnil;
+}
+
+static VALUE stream_mark_pop(const VALUE self) {
+    UNWRAP_STREAM;
+    if (s->marks_idx == 0) {
+        rb_raise(rb_eRuntimeError, "BUG: No mark to pop");
+    }
+    s->marks_idx--;
+    return Qnil;
+}
+
+static bool is_eof(const VALUE v) {
+    const VALUE k = rb_funcall(v, rb_intern("kind"), 0);
+    Check_Type(k, T_SYMBOL);
+    return rb_sym2id(k) == id_type_eof;
+}
+
+static VALUE stream_is_empty(const VALUE self) {
+    UNWRAP_STREAM;
+    if (s->tokens_idx >= s->tokens_len || is_eof(s->look[0]))
+        return Qtrue;
+    return Qfalse;
+}
+
+static VALUE stream_peek_kind(const VALUE self) {
+    UNWRAP_STREAM;
+    const VALUE k = rb_funcall(s->look[0], rb_intern("kind"), 0);
+    Check_Type(k, T_SYMBOL);
+    return k;
+}
+
+static VALUE stream_status(const VALUE self) {
+    UNWRAP_STREAM;
+    const VALUE h = rb_hash_new();
+    rb_hash_aset(h, ID2SYM(rb_intern("eof")), s->eof);
+    rb_hash_aset(h, ID2SYM(rb_intern("tokens")), s->tokens);
+    rb_hash_aset(h, ID2SYM(rb_intern("tokens_idx")), INT2NUM(s->tokens_idx));
+    rb_hash_aset(h, ID2SYM(rb_intern("tokens_len")), INT2NUM((int)s->tokens_len));
+    rb_hash_aset(h, ID2SYM(rb_intern("look0")), s->look[0]);
+    rb_hash_aset(h, ID2SYM(rb_intern("look1")), s->look[1]);
+    rb_hash_aset(h, ID2SYM(rb_intern("marks_idx")), INT2NUM(s->marks_idx));
+    return h;
+}
+
+void Init_minicss_token_stream(void) {
+    const VALUE mMiniCSS = rb_define_module("MiniCSS");
+    const VALUE mCSS = rb_define_module_under(mMiniCSS, "CSS");
+    const VALUE cStream = rb_define_class_under(mCSS, "TokenStream", rb_cObject);
+    cTokenClass = rb_path2class("MiniCSS::CSS::Token");
+
+    INITIALIZE_REUSABLE_SYMBOL(whitespace);
+    INITIALIZE_REUSABLE_SYMBOL(eof);
+
+    rb_define_alloc_func(cStream, stream_alloc);
+    rb_define_method(cStream, "initialize", stream_initialize, 1);
+    rb_define_method(cStream, "peek", stream_peek, 0);
+    rb_define_method(cStream, "peek_kind", stream_peek_kind, 0);
+    rb_define_method(cStream, "consume", stream_consume, 0);
+    rb_define_method(cStream, "discard_whitespace", stream_discard_whitespace, 0);
+    rb_define_method(cStream, "discard", stream_discard, 0);
+
+    rb_define_method(cStream, "mark", stream_create_mark, 0);
+    rb_define_method(cStream, "restore", stream_mark_restore, 0);
+    rb_define_method(cStream, "pop", stream_mark_pop, 0);
+    rb_define_method(cStream, "empty?", stream_is_empty, 0);
+    rb_define_method(cStream, "status", stream_status, 0);
+}

data/lib/minicss/css/parser.rb

@@ -5,8 +5,6 @@ module MiniCSS
     class Parser
       using StringRefinements
 
-      attr_reader :stream
-
       def initialize(tokens)
         @stream = TokenStream.new(tokens)
         @tokens = tokens
@@ -27,25 +25,25 @@ module MiniCSS
       end
 
       def parse_rule
-        stream.discard_whitespace
-        return SyntaxError.new("empty") if stream.empty?
+        @stream.discard_whitespace
+        return SyntaxError.new("empty") if @stream.empty?
 
-        result = if stream.peek.kind == :at_keyword
+        result = if @stream.peek.kind == :at_keyword
           consume_at_rule
         else
           consume_qualified_rule
         end
         return SyntaxError.new("invalid") unless result
 
-        stream.discard_whitespace
-        return SyntaxError.new("extra-input") unless stream.empty?
+        @stream.discard_whitespace
+        return SyntaxError.new("extra-input") unless @stream.empty?
 
         result
       end
 
       def parse_declaration
-        stream.discard_whitespace
-        return SyntaxError.new("empty") if stream.empty?
+        @stream.discard_whitespace
+        return SyntaxError.new("empty") if @stream.empty?
 
         decl = consume_declaration
         return SyntaxError.new("invalid") unless decl
@@ -54,25 +52,23 @@ module MiniCSS
       end
 
       def parse_component_value
-        stream.discard_whitespace
-        return SyntaxError.new("empty") if stream.empty?
+        @stream.discard_whitespace
+        return SyntaxError.new("empty") if @stream.empty?
 
         value = consume_component_value
-        stream.discard_whitespace
-        return SyntaxError.new("extra-input") unless stream.empty?
+        @stream.discard_whitespace
+        return SyntaxError.new("extra-input") unless @stream.empty?
 
         value
       end
 
-      def parse_component_value_list
-        consume_component_value_list
-      end
+      def parse_component_value_list = consume_component_value_list
 
       def parse_component_value_comma_list
         groups = []
-        until stream.empty?
+        until @stream.empty?
           groups << consume_component_value_list(stop: :comma)
-          stream.discard
+          @stream.discard
         end
         groups
       end
@@ -80,7 +76,7 @@ module MiniCSS
       # Helpers
 
       def assert_next_token(kind)
-        stream.peek.kind == kind
+        @stream.peek.kind == kind
       end
 
       # Parsers
@@ -88,9 +84,9 @@ module MiniCSS
       def consume_stylesheet_contents
         rules = []
         loop do
-          case stream.peek.kind
+          case @stream.peek.kind
           when :whitespace, :cdo, :cdc
-            stream.discard
+            @stream.discard
 
           when :eof
             return rules
@@ -109,11 +105,11 @@ module MiniCSS
       def consume_at_rule(nested: false)
         return unless assert_next_token(:at_keyword)
 
-        rule = AST::AtRule.new(name: stream.consume)
+        rule = AST::AtRule.new(name: @stream.consume)
         loop do
-          case stream.peek.kind
+          case @stream.peek.kind
           when :semicolon, :eof
-            stream.discard
+            @stream.discard
             # TODO: If rule is valid in the current context, return it; otherwise return nothing.
             return rule
 
@@ -123,10 +119,10 @@ module MiniCSS
               return rule
             end
 
-            rule.prelude << consume
+            rule.prelude << @stream.consume
 
           when :left_curly
-            consume_block.each { rule.child_rules << it }
+            rule.child_rules.append(*consume_block)
             return rule
 
           else
@@ -139,14 +135,14 @@ module MiniCSS
         rule = AST::QualifiedRule.new
 
         loop do
-          case stream.peek.kind
+          case @stream.peek.kind
           when :eof, stop
             return nil
 
           when :right_curly
             return nil if nested
 
-            rule.prelude << consume
+            rule.prelude << @stream.consume
 
           when :left_curly
             non_ws = rule.prelude.reject { it.kind == :whitespace }
@@ -179,9 +175,9 @@ module MiniCSS
       def consume_block
         return unless assert_next_token(:left_curly)
 
-        stream.discard
+        @stream.discard
         rules = consume_block_contents
-        stream.discard
+        @stream.discard
         rules
       end
 
@@ -190,9 +186,9 @@ module MiniCSS
         decls = AST::DeclarationList.new
 
         loop do
-          case stream.peek.kind
+          case @stream.peek.kind
           when :whitespace, :semicolon
-            stream.discard
+            @stream.discard
           when :eof, :right_curly
             rules << decls unless decls.empty?
             return rules
@@ -204,15 +200,15 @@ module MiniCSS
             at = consume_at_rule(nested: true)
             rules << at if at.is_a? AST::AtRule
           else
-            stream.mark
+            @stream.mark
             decl = consume_declaration(nested: true)
             if decl.is_a? AST::Declaration
               decls << decl
-              stream.pop
+              @stream.pop
               next
             end
 
-            stream.restore
+            @stream.restore
             rule = consume_qualified_rule(nested: true)
             if rule.is_a? InvalidRuleError
               unless decls.empty?
@@ -232,23 +228,23 @@ module MiniCSS
 
       def consume_declaration(nested: false)
         decl = AST::Declaration.new
-        if stream.peek.kind == :ident
-          decl.name = stream.consume
+        if @stream.peek.kind == :ident
+          decl.name = @stream.consume
         else
           consume_bad_declaration(nested:)
           return nil
         end
 
-        stream.discard_whitespace
+        @stream.discard_whitespace
 
-        if stream.peek.kind == :colon
-          stream.discard
+        if @stream.peek.kind == :colon
+          @stream.discard
         else
           consume_bad_declaration(nested:)
           return nil
         end
 
-        stream.discard_whitespace
+        @stream.discard_whitespace
 
         decl.value = consume_component_value_list(nested:, stop: :semicolon)
 
@@ -276,14 +272,14 @@ module MiniCSS
 
       def consume_bad_declaration(nested:)
         loop do
-          case stream.peek.kind
+          case @stream.peek.kind
           when :eof, :semicolon
-            stream.discard
+            @stream.discard
             return nil
           when :right_curly
             return if nested
 
-            stream.discard
+            @stream.discard
           else
             consume_component_value
           end
@@ -293,14 +289,14 @@ module MiniCSS
       def consume_component_value_list(stop: nil, nested: false)
         values = []
         loop do
-          case stream.peek.kind
+          case @stream.peek.kind
           when :eof, stop
             return values
 
           when :right_curly
             return values if nested
 
-            values << stream.consume
+            values << @stream.consume
 
           else
             values << consume_component_value
@@ -309,20 +305,20 @@ module MiniCSS
       end
 
       def consume_component_value
-        case stream.peek.kind
+        case @stream.peek_kind
         when :left_curly, :left_square_bracket, :left_parenthesis
           consume_simple_block
         when :function
           consume_function
         else
-          stream.consume
+          @stream.consume
         end
       end
 
       def consume_simple_block
         return if !assert_next_token(:left_curly) && !assert_next_token(:left_square_bracket) && !assert_next_token(:left_parenthesis)
 
-        block = AST::SimpleBlock.new(associated_token: stream.consume.kind)
+        block = AST::SimpleBlock.new(associated_token: @stream.consume.kind)
         ending_token = {
           left_curly: :right_curly,
           left_parenthesis: :right_parenthesis,
@@ -330,9 +326,9 @@ module MiniCSS
         }[block.associated_token]
 
         loop do
-          case stream.peek.kind
+          case @stream.peek.kind
           when :eof, ending_token
-            stream.discard
+            @stream.discard
             return block
           else
             block.value << consume_component_value
@@ -343,12 +339,12 @@ module MiniCSS
       def consume_function
         return unless assert_next_token(:function)
 
-        func = AST::Function.new(name: stream.consume)
+        func = AST::Function.new(name: @stream.consume)
 
         loop do
-          case stream.peek.kind
+          case @stream.peek.kind
           when :eof, :right_parenthesis
-            stream.discard
+            @stream.discard
             return func
           else
             func.value << consume_component_value