deadfire 0.2.0 → 0.3.0

data/lib/deadfire/front_end/parser.rb

@@ -0,0 +1,156 @@
+# frozen_string_literal: true
+
+module Deadfire
+  module FrontEnd
+    class Parser
+      attr_reader :error_reporter, :tokens, :options, :current
+
+      def initialize(tokens, error_reporter)
+        @error_reporter = error_reporter
+        @tokens = tokens
+        @current = 0
+        @stylesheet = StylesheetNode.new
+      end
+
+      def parse
+        # top level it's a list of statements
+        # statements are either rules or at-rules
+        # rules are selectors + declarations
+        # at-rules are at-keyword + block
+        # block is a list of declarations?
+        # declarations are property + value
+        while !is_at_end?
+          if check(:comment)
+            comment = add_comment
+            @stylesheet << comment if Deadfire.configuration.keep_comments
+          elsif check(:newline)
+            newline = add_newline
+            @stylesheet << newline if Deadfire.configuration.keep_whitespace
+          elsif matches_at_rule?
+            @stylesheet << at_rule_declaration
+          else
+            @stylesheet << ruleset_declaration
+          end
+        end
+        @stylesheet
+      end
+
+      private
+
+      def is_at_end?
+        peek.type == :eof
+      end
+
+      def peek
+        tokens[current]
+      end
+
+      def previous
+        tokens[current - 1]
+      end
+
+      def advance
+        @current += 1 unless is_at_end?
+        previous
+      end
+
+      def check(type)
+        return false if is_at_end?
+
+        peek.type == type
+      end
+
+      def match?(*types)
+        types.each do |type|
+          if check(type)
+            advance
+            return true
+          end
+        end
+
+        false
+      end
+
+      def consume(type, message)
+        if check(type)
+          advance
+          return
+        end
+
+        error_reporter.error(peek.lineno, message)
+      end
+
+      def matches_at_rule?
+        check(:at_rule)
+      end
+
+      def matches_nested_rule?
+        match?(:ampersand)
+      end
+
+      def parse_block
+        block = BlockNode.new
+        block << previous
+
+        while !is_at_end?
+          if match?(:right_brace)
+            break
+          elsif matches_at_rule?
+            block << at_rule_declaration
+          elsif match?(:left_brace)
+            block << parse_block
+          else
+            block << advance
+          end
+        end
+
+        block << previous
+        block
+      end
+
+      def ruleset_declaration
+        values = []
+        while !match?(:left_brace)
+          values << advance
+        end
+
+        selector = SelectorNode.new(values[0..-1])
+
+        block = parse_block
+        RulesetNode.new(selector, block)
+      end
+
+      def add_comment
+        consume(:comment, "Expect comment")
+        CommentNode.new(previous)
+      end
+
+      def add_newline
+        consume(:newline, "Expect newline")
+        NewlineNode.new(previous.lexeme)
+      end
+
+      def at_rule_declaration
+        consume(:at_rule, "Expect at rule")
+        keyword = previous
+
+        # peek until we get to ; or {, if we reach ; then add to at rule node and return
+        values = []
+        while !match?(:semicolon, :left_brace) && !is_at_end?
+          values << advance
+        end
+
+        if previous.type == :semicolon
+          if keyword.lexeme == "@apply"
+            return ApplyNode.new(keyword, values)
+          else
+            values << previous # add the semicolon to the values
+            return AtRuleNode.new(keyword, values, nil)
+          end
+        end
+
+        AtRuleNode.new(keyword, values[0..-1], parse_block) # remove the left brace, because it's not a value, but part of the block
+      end
+    end
+  end
+end

data/lib/deadfire/front_end/ruleset_node.rb

@@ -0,0 +1,18 @@
+# frozen_string_literal: true
+
+module Deadfire
+  module FrontEnd
+    class RulesetNode < BaseNode
+      attr_reader :selector, :block
+
+      def initialize(selector, block)
+        @selector = selector
+        @block = block
+      end
+
+      def accept(visitor)
+        visitor.visit_ruleset_node(self)
+      end
+    end
+  end
+end

data/lib/deadfire/front_end/scanner.rb

@@ -0,0 +1,266 @@
+# frozen_string_literal: true
+
+module Deadfire
+  module FrontEnd
+    class Scanner
+      def initialize(source, error_reporter)
+        @source = source
+        @total_chars = @source.length
+        @tokens = []
+        @start = 0
+        @current = 0
+        @line = 1
+        @error_reporter = error_reporter
+      end
+
+      def tokenize
+        until at_end?
+          reset_counter
+          scan_token
+        end
+
+        @tokens << Token.new(:eof, "", nil, @line)
+      end
+
+      private
+
+      NEWLINE = "\n"
+
+      def reset_counter
+        @start = @current - 1
+      end
+
+      def at_end?
+        @current >= @total_chars
+      end
+
+      def scan_token
+        token = advance
+        case token
+        when "@" then add_at_rule
+        when "{" then add_token(:left_brace)
+        when "}" then add_token(:right_brace)
+        when "#" then add_token(:id_selector)
+        when "." then add_token(:class_selector)
+        when ";" then add_token(:semicolon)
+        when "," then add_token(:comma)
+        when "(" then add_token(:left_paren)
+        when ")" then add_token(:right_paren)
+        when "_" then add_token(:underscore)
+        when "=" then add_token(:equal)
+        when "~" then add_token(:tilde)
+        when "*" then add_token(:asterisk)
+        when "&" then add_token(:ampersand)
+        when ":" then add_psuedo_selector
+        when "-" then add_hypen_token
+        when "/" then add_forward_slash_or_comment
+        when "'" then add_token(:single_quote)
+        when NEWLINE then add_newline_token
+        when " ", "\r", "\t" then add_whitespace_token
+        when '"' then add_string_token
+        else
+          if digit?(token)
+            add_number_token
+          elsif text?(token)
+            add_text_token # or word token?
+          else
+            add_token(:other)
+          end
+        end
+      end
+
+      def add_token(type, literal = nil)
+        text = @source[@start + 1..current_char_position]
+        @tokens << Token.new(type, text, literal, @line)
+      end
+
+      def add_at_rule(literal = nil)
+        selector = [current_char]
+
+        while Spec::CSS_AT_RULES.none? { |kwrd| kwrd == selector.join + peek } && !at_end?
+          break if peek == NEWLINE
+          selector << advance
+        end
+
+        # final char in at-rule
+        selector << advance
+
+        current_at_rule = selector.join
+        at_rule = Spec::CSS_AT_RULES.find { |kwrd| kwrd == current_at_rule }
+
+        if peek == NEWLINE
+          @line += 1
+          @error_reporter.error(@line, "at-rule cannot be on multiple lines.")
+          add_token(:at_rule, current_at_rule)
+        elsif at_rule
+          token = add_token(:at_rule, "at_#{at_rule[1..-1]}")
+          if at_rule == "@import"
+            prescan_import_rule(token.last)
+          else
+            token
+          end
+        else
+          @error_reporter.error(@line, "Invalid at-rule.")
+        end
+      end
+
+      def add_string_token
+        while peek != '"' && !at_end?
+          @line += 1 if peek == NEWLINE
+          advance
+        end
+
+        if at_end?
+          @error_reporter.error(@line, "Unterminated string.")
+          return
+        end
+
+        advance
+
+        # Trim the surrounding quotes.
+        # TODO: this does not look right... page 50 crafting interpreters.
+        value = @source[@start + 2..current_char_position]
+        add_token(:string, value)
+      end
+
+      # This token is very similar to the string token, but we want to explicitly
+      # split up text from string, because string in css is surrounded by quotes and text is free form
+      # which can be a property or value e.g. `color: red;`.
+      def add_text_token
+        while text?(peek) && !at_end?
+          advance
+        end
+
+        value = @source[@start+1..current_char_position]
+        add_token(:text, value)
+      end
+
+      def add_number_token
+        while digit?(peek)
+          advance
+        end
+
+        # Look for a fractional part.
+        if peek == "." && digit?(peek_next)
+          # Consume the "."
+          advance
+
+          while digit?(peek)
+            advance
+          end
+        end
+
+        add_token(:number, @source[@start..@current].to_f)
+      end
+
+      def add_psuedo_selector
+        if peek == ":"
+          advance
+          add_token(:double_colon)
+        else
+          add_token(:colon)
+        end
+      end
+
+      def add_hypen_token
+        if peek == "-"
+          advance
+          add_token(:double_hyphen)
+        else
+          add_token(:hyphen)
+        end
+      end
+
+      def add_forward_slash_or_comment
+        if peek == "*"
+          advance # consume the *
+          while peek != "*" && peek_next != "/" && !at_end?
+            @line += 1 if peek == NEWLINE
+            advance
+          end
+
+          if at_end? && peek != "*"
+            @error_reporter.error(@line, "Unterminated comment on line #{@line}.")
+          else
+            advance # consume the *
+            advance # consume the /
+          end
+          add_token(:comment) # Add the comment anyway, but report an error.
+        else
+          add_token(:forward_slash)
+        end
+      end
+
+      def add_whitespace_token
+        add_token(:whitespace) if Deadfire.configuration.keep_whitespace
+      end
+
+      def add_newline_token
+        @line += 1
+        add_token(:newline) if Deadfire.configuration.keep_whitespace
+      end
+
+      def current_char_position
+        @current - 1
+      end
+
+      def current_char
+        @source[current_char_position]
+      end
+
+      def advance
+        @current += 1
+        current_char
+      end
+
+      def peek
+        @source[@current] unless at_end?
+      end
+
+      def peek_next
+        @source[@current + 1] unless at_end?
+      end
+
+      def digit?(char)
+        char >= "0" && char <= "9"
+      end
+
+      def text?(char)
+        (char >= "a" && char <= "z") || (char >= "A" && char <= "Z")
+      end
+
+      def prescan_import_rule(token)
+        # we want to get all the text between the @import and the semicolon
+        # so we can parse the file and add it to the ast
+        reset_counter
+
+        while peek != ";" && !at_end?
+          advance
+        end
+
+        if at_end?
+          @error_reporter.error(@line, "Unterminated import rule.")
+          return
+        end
+
+        add_token(:text)
+        advance # remove the semicolon
+
+        text_token = @tokens.last
+
+        text = text_token.lexeme.gsub(/\\|"/, '')
+        file = FilenameHelper.resolve_import_path(text, @line)
+
+        # file is ready for scanning
+        content = File.read(file)
+        scanner = Scanner.new(content, @error_reporter)
+
+        @tokens.pop # remove the text token
+        @tokens.pop # remove the at_rule token
+
+        imported_tokens = scanner.tokenize[0..-2]
+        @tokens.concat imported_tokens
+      end
+    end
+  end
+end

data/lib/deadfire/front_end/selector_node.rb

@@ -0,0 +1,52 @@
+# frozen_string_literal: true
+
+module Deadfire
+  module FrontEnd
+    class SelectorNode < BaseNode
+      attr_reader :selector, :mixin_name
+
+      def initialize(tokens)
+        @selector = tokens_to_selector(tokens)
+        @mixin_name = fetch_mixin_name_from(tokens)
+      end
+
+      def accept(visitor)
+        visitor.visit_selector_node(self)
+      end
+
+      def cacheable?
+        selector.start_with?(".")
+      end
+
+      private
+
+      # TODO:
+      # For descendant values such as `a b`, we need to add a space between the tokens,
+      # otherwise all other values will be concatenated together.
+      def tokens_to_selector(tokens)
+        tokens.map(&:lexeme).join("").strip
+      end
+
+      # https://sass-lang.com/guide
+      # https://sass-lang.com/documentation/style-rules/selector
+      # TODO: this needs some tests and a lot more work
+      # not all selectors are valid mixin names
+      def fetch_mixin_name_from(tokens)
+        @_cached_mixin_name ||= begin
+          name = []
+          tokens.each do |token|
+            case token.type
+            when :right_paren, :left_paren
+              break
+            when :colon
+              name << token.lexeme
+            else
+              name << token.lexeme
+            end
+          end
+          name.join("")
+        end
+      end
+    end
+  end
+end

data/lib/deadfire/front_end/stylesheet_node.rb

@@ -0,0 +1,21 @@
+# frozen_string_literal: true
+
+module Deadfire
+  module FrontEnd
+    class StylesheetNode
+      attr_reader :statements
+
+      def initialize
+        @statements = []
+      end
+
+      def accept(visitor)
+        visitor.visit_stylesheet_node(self)
+      end
+
+      def << (node)
+        @statements << node
+      end
+    end
+  end
+end

data/lib/deadfire/front_end/token.rb

@@ -0,0 +1,20 @@
+# frozen_string_literal: true
+
+module Deadfire
+  module FrontEnd
+    class Token
+      attr_reader :type, :lexeme, :literal, :lineno
+
+      def initialize(type, lexeme, literal, lineno)
+        @type = type
+        @lexeme = lexeme
+        @literal = literal
+        @lineno = lineno
+      end
+
+      def to_s
+        "[:#{type}] #{lexeme}"
+      end
+    end
+  end
+end

data/lib/deadfire/interpreter.rb

@@ -0,0 +1,88 @@
+# Frozen_string_literal: true
+
+module Deadfire
+  class Interpreter # :nodoc:
+    singleton_class.attr_accessor :cached_apply_rules
+    self.cached_apply_rules = Hash.new { |h, k| h[k] = nil }
+
+    def initialize(error_reporter)
+      @error_reporter = error_reporter
+    end
+
+    def interpret(node)
+      node.accept(self)
+    end
+
+    def visit_stylesheet_node(node)
+      node.statements.each { |child| child.accept(self) }
+    end
+
+    def visit_at_rule_node(node)
+      if node.block
+        visit_block_node(node.block, node)
+      end
+    end
+
+    def visit_ruleset_node(node)
+      if node.block
+        visit_block_node(node.block, node)
+
+        unless Interpreter.cached_apply_rules[node.selector.selector]
+          Interpreter.cached_apply_rules[node.selector.selector] = node.block if node.selector.cacheable?
+        end
+      end
+    end
+
+    def visit_block_node(node, parent)
+      node.declarations.each do |declaration|
+        case declaration
+        when ApplyNode
+          apply_mixin(declaration, node)
+        else
+          # we may not need to visit this as we don't process/transform/optimize
+        end
+      end
+    end
+
+    def visit_declaration_node(node)
+      node.accept(self)
+    end
+
+    def visit_comment_node(node)
+      # node.accept(self)
+    end
+
+    def visit_apply_node(node)
+      # do nothing for now
+    end
+
+    def visit_newline_node(node)
+    end
+
+    private
+
+    def apply_mixin(mixin, node)
+      updated_declarations = []
+      mixin.mixin_names.each do |mixin_name|
+        if Interpreter.cached_apply_rules[mixin_name]
+          cached_block = Interpreter.cached_apply_rules[mixin_name]
+
+          # NOTE: remove the left and right brace but we probably don't need to do this, how can this be simplified?
+          cached_block.declarations[1...-1].each do |cached_declaration|
+            updated_declarations << cached_declaration
+          end
+          updated_declarations.shift if updated_declarations.first.type == :newline
+          updated_declarations.pop if updated_declarations.last.type == :newline
+        else
+          @error_reporter.error(mixin.lineno, "Mixin #{mixin_name} not found") # TODO: we need the declarations lineno, not the block
+        end
+      end
+
+      if updated_declarations.any?
+        index = node.declarations.index(mixin)
+        node.declarations.delete_at(index)
+        node.declarations.insert(index, *updated_declarations)
+      end
+    end
+  end
+end