mui 0.1.0 → 0.3.0

data/lib/mui/syntax/lexers/html_lexer.rb

@@ -0,0 +1,118 @@
+# frozen_string_literal: true
+
+module Mui
+  module Syntax
+    module Lexers
+      # Lexer for HTML source files
+      class HtmlLexer < LexerBase
+        # Pre-compiled patterns with \G anchor for position-specific matching
+        COMPILED_PATTERNS = [
+          # HTML comment (single line)
+          [:comment, /\G<!--.*?-->/],
+          # DOCTYPE declaration
+          [:preprocessor, /\G<!DOCTYPE[^>]*>/i],
+          # CDATA section
+          [:string, /\G<!\[CDATA\[.*?\]\]>/],
+          # Closing tag
+          [:keyword, %r{\G</[a-zA-Z][a-zA-Z0-9-]*\s*>}],
+          # Self-closing tag
+          [:keyword, %r{\G<[a-zA-Z][a-zA-Z0-9-]*(?:\s+[^>]*)?/>}],
+          # Opening tag with attributes
+          [:keyword, /\G<[a-zA-Z][a-zA-Z0-9-]*(?=[\s>])/],
+          # Tag closing bracket
+          [:keyword, /\G>/],
+          # Attribute name
+          [:type, /\G[a-zA-Z][a-zA-Z0-9_-]*(?==)/],
+          # Double quoted attribute value
+          [:string, /\G"[^"]*"/],
+          # Single quoted attribute value
+          [:string, /\G'[^']*'/],
+          # Unquoted attribute value (limited characters)
+          [:string, /\G=[^\s>"']+/],
+          # HTML entities
+          [:constant, /\G&(?:#\d+|#x[0-9a-fA-F]+|[a-zA-Z]+);/],
+          # Equal sign (for attributes)
+          [:operator, /\G=/]
+        ].freeze
+
+        # Multiline comment patterns
+        COMMENT_START = /<!--/
+        COMMENT_END = /-->/
+        COMMENT_START_ANCHOR = /\G<!--/
+
+        protected
+
+        def compiled_patterns
+          COMPILED_PATTERNS
+        end
+
+        # Handle multiline HTML comments
+        def handle_multiline_state(line, pos, state)
+          return [nil, nil, pos] unless state == :html_comment
+
+          end_match = line[pos..].match(COMMENT_END)
+          if end_match
+            end_pos = pos + end_match.begin(0) + 2 # --> is 3 chars
+            text = line[pos..end_pos]
+            token = Token.new(
+              type: :comment,
+              start_col: pos,
+              end_col: end_pos,
+              text:
+            )
+            [token, nil, end_pos + 1]
+          else
+            text = line[pos..]
+            token = if text.empty?
+                      nil
+                    else
+                      Token.new(
+                        type: :comment,
+                        start_col: pos,
+                        end_col: line.length - 1,
+                        text:
+                      )
+                    end
+            [token, :html_comment, line.length]
+          end
+        end
+
+        def check_multiline_start(line, pos)
+          rest = line[pos..]
+
+          # Check for <!-- that doesn't have --> on this line
+          start_match = rest.match(COMMENT_START)
+          return [nil, nil, pos] unless start_match
+
+          start_pos = pos + start_match.begin(0)
+          after_start = line[(start_pos + 4)..] # Skip <!--
+
+          if after_start&.include?("-->")
+            [nil, nil, pos]
+          else
+            text = line[start_pos..]
+            token = Token.new(
+              type: :comment,
+              start_col: start_pos,
+              end_col: line.length - 1,
+              text:
+            )
+            [:html_comment, token, line.length]
+          end
+        end
+
+        private
+
+        def match_token(line, pos)
+          # Check for start of multiline comment
+          if line[pos..].match?(COMMENT_START_ANCHOR)
+            rest = line[(pos + 4)..]
+            return nil unless rest&.include?("-->")
+          end
+
+          super
+        end
+      end
+    end
+  end
+end

data/lib/mui/syntax/lexers/javascript_lexer.rb

@@ -0,0 +1,197 @@
+# frozen_string_literal: true
+
+module Mui
+  module Syntax
+    module Lexers
+      # Lexer for JavaScript source code
+      class JavaScriptLexer < LexerBase
+        # JavaScript keywords
+        KEYWORDS = %w[
+          async await break case catch class const continue debugger default
+          delete do else export extends finally for function if import in
+          instanceof let new return static switch throw try typeof var void
+          while with yield
+        ].freeze
+
+        # JavaScript built-in types and values
+        CONSTANTS = %w[true false null undefined NaN Infinity this super].freeze
+
+        # Pre-compiled patterns with \G anchor for position-specific matching
+        COMPILED_PATTERNS = [
+          # Single line comment
+          [:comment, %r{\G//.*}],
+          # Single-line block comment /* ... */ on one line
+          [:comment, %r{\G/\*.*?\*/}],
+          # Template literal (single line)
+          [:string, /\G`[^`]*`/],
+          # Double quoted string (with escape handling)
+          [:string, /\G"(?:[^"\\]|\\.)*"/],
+          # Single quoted string (with escape handling)
+          [:string, /\G'(?:[^'\\]|\\.)*'/],
+          # Regular expression literal
+          [:regex, %r{\G/(?:[^/\\]|\\.)+/[gimsuy]*}],
+          # Float numbers (must be before integer)
+          [:number, /\G\b\d+\.\d+(?:e[+-]?\d+)?\b/i],
+          # Hexadecimal
+          [:number, /\G\b0x[0-9a-fA-F]+n?\b/i],
+          # Octal
+          [:number, /\G\b0o[0-7]+n?\b/i],
+          # Binary
+          [:number, /\G\b0b[01]+n?\b/i],
+          # Integer (with optional BigInt suffix)
+          [:number, /\G\b\d+n?\b/],
+          # Constants (true, false, null, undefined, NaN, Infinity, this, super)
+          [:constant, /\G\b(?:true|false|null|undefined|NaN|Infinity|this|super)\b/],
+          # Keywords
+          [:keyword, /\G\b(?:async|await|break|case|catch|class|const|continue|debugger|default|delete|do|else|export|extends|finally|for|function|if|import|in|instanceof|let|new|return|static|switch|throw|try|typeof|var|void|while|with|yield)\b/],
+          # Class/constructor names (start with uppercase)
+          [:constant, /\G\b[A-Z][a-zA-Z0-9_]*\b/],
+          # Regular identifiers
+          [:identifier, /\G\b[a-zA-Z_$][a-zA-Z0-9_$]*\b/],
+          # Operators (=>must come before = patterns)
+          [:operator, %r{\G(?:\.{3}|=>|&&=?|\|\|=?|\?\?=?|===?|!==?|>>>?=?|<<=?|\+\+|--|\?\.?|[+\-*/%&|^<>=!]=?)}]
+        ].freeze
+
+        # Multiline patterns (pre-compiled)
+        BLOCK_COMMENT_END = %r{\*/}
+        BLOCK_COMMENT_START = %r{/\*}
+        BLOCK_COMMENT_START_ANCHOR = %r{\A/\*}
+
+        # Template literal patterns
+        TEMPLATE_LITERAL_START = /\A`/
+        TEMPLATE_LITERAL_END = /`/
+
+        protected
+
+        def compiled_patterns
+          COMPILED_PATTERNS
+        end
+
+        # Handle multiline constructs
+        def handle_multiline_state(line, pos, state)
+          case state
+          when :block_comment
+            handle_block_comment(line, pos)
+          when :template_literal
+            handle_template_literal(line, pos)
+          else
+            [nil, nil, pos]
+          end
+        end
+
+        def check_multiline_start(line, pos)
+          rest = line[pos..]
+
+          # Check for template literal start
+          if rest.match?(TEMPLATE_LITERAL_START)
+            after_start = line[(pos + 1)..]
+            unless after_start&.include?("`")
+              text = line[pos..]
+              token = Token.new(
+                type: :string,
+                start_col: pos,
+                end_col: line.length - 1,
+                text:
+              )
+              return [:template_literal, token, line.length]
+            end
+          end
+
+          # Check for /* that doesn't have a matching */ on this line
+          start_match = rest.match(BLOCK_COMMENT_START)
+          return [nil, nil, pos] unless start_match
+
+          start_pos = pos + start_match.begin(0)
+          after_start = line[(start_pos + 2)..]
+
+          if after_start&.include?("*/")
+            [nil, nil, pos]
+          else
+            text = line[start_pos..]
+            token = Token.new(
+              type: :comment,
+              start_col: start_pos,
+              end_col: line.length - 1,
+              text:
+            )
+            [:block_comment, token, line.length]
+          end
+        end
+
+        private
+
+        def handle_block_comment(line, pos)
+          end_match = line[pos..].match(BLOCK_COMMENT_END)
+          if end_match
+            end_pos = pos + end_match.begin(0) + 1
+            text = line[pos..end_pos]
+            token = Token.new(
+              type: :comment,
+              start_col: pos,
+              end_col: end_pos,
+              text:
+            )
+            [token, nil, end_pos + 1]
+          else
+            text = line[pos..]
+            token = if text.empty?
+                      nil
+                    else
+                      Token.new(
+                        type: :comment,
+                        start_col: pos,
+                        end_col: line.length - 1,
+                        text:
+                      )
+                    end
+            [token, :block_comment, line.length]
+          end
+        end
+
+        def handle_template_literal(line, pos)
+          end_match = line[pos..].match(TEMPLATE_LITERAL_END)
+          if end_match
+            end_pos = pos + end_match.begin(0)
+            text = line[pos..end_pos]
+            token = Token.new(
+              type: :string,
+              start_col: pos,
+              end_col: end_pos,
+              text:
+            )
+            [token, nil, end_pos + 1]
+          else
+            text = line[pos..]
+            token = if text.empty?
+                      nil
+                    else
+                      Token.new(
+                        type: :string,
+                        start_col: pos,
+                        end_col: line.length - 1,
+                        text:
+                      )
+                    end
+            [token, :template_literal, line.length]
+          end
+        end
+
+        def match_token(line, pos)
+          # Check for start of template literal
+          if line[pos..].match?(TEMPLATE_LITERAL_START)
+            rest = line[(pos + 1)..]
+            return nil unless rest&.include?("`")
+          end
+
+          # Check for start of multiline comment
+          if line[pos..].match?(BLOCK_COMMENT_START_ANCHOR)
+            rest = line[(pos + 2)..]
+            return nil unless rest&.include?("*/")
+          end
+
+          super
+        end
+      end
+    end
+  end
+end

data/lib/mui/syntax/lexers/markdown_lexer.rb

@@ -0,0 +1,210 @@
+# frozen_string_literal: true
+
+module Mui
+  module Syntax
+    module Lexers
+      # Lexer for Markdown source files
+      class MarkdownLexer < LexerBase
+        # Pre-compiled patterns with \G anchor for position-specific matching
+        # Markdown is line-oriented, so we check line-start patterns separately
+        COMPILED_PATTERNS = [
+          # Inline code (backtick)
+          [:string, /\G`[^`]+`/],
+          # Bold with asterisks
+          [:keyword, /\G\*\*[^*]+\*\*/],
+          # Bold with underscores
+          [:keyword, /\G__[^_]+__/],
+          # Italic with asterisks
+          [:comment, /\G\*[^*]+\*/],
+          # Italic with underscores
+          [:comment, /\G_[^_]+_/],
+          # Strikethrough
+          [:comment, /\G~~[^~]+~~/],
+          # Link [text](url)
+          [:constant, /\G\[[^\]]+\]\([^)]+\)/],
+          # Image ![alt](url)
+          [:constant, /\G!\[[^\]]*\]\([^)]+\)/],
+          # Reference link [text][ref]
+          [:constant, /\G\[[^\]]+\]\[[^\]]*\]/],
+          # Autolink <url> or <email>
+          [:constant, /\G<[a-zA-Z][a-zA-Z0-9+.-]*:[^>]+>/],
+          # HTML tags
+          [:preprocessor, %r{\G</?[a-zA-Z][a-zA-Z0-9]*[^>]*>}]
+        ].freeze
+
+        # Line-start patterns (checked at beginning of line)
+        HEADING_PATTERN = /\A(\#{1,6})\s+(.*)$/
+        BLOCKQUOTE_PATTERN = /\A>\s*/
+        UNORDERED_LIST_PATTERN = /\A\s*[-*+]\s+/
+        ORDERED_LIST_PATTERN = /\A\s*\d+\.\s+/
+        HORIZONTAL_RULE_PATTERN = /\A([-*_])\s*\1\s*\1[\s\1]*$/
+        CODE_FENCE_START = /\A```(\w*)/
+        CODE_FENCE_END = /\A```\s*$/
+        LINK_DEFINITION_PATTERN = /\A\s*\[[^\]]+\]:\s+\S+/
+
+        # Override tokenize to handle line-start patterns
+        def tokenize(line, state = nil)
+          tokens = []
+          pos = 0
+
+          # Handle code fence state
+          if state == :code_fence
+            if line.match?(CODE_FENCE_END)
+              token = Token.new(
+                type: :string,
+                start_col: 0,
+                end_col: line.length - 1,
+                text: line
+              )
+              return [[token], nil]
+            else
+              unless line.empty?
+                token = Token.new(
+                  type: :string,
+                  start_col: 0,
+                  end_col: line.length - 1,
+                  text: line
+                )
+              end
+              return [token ? [token] : [], :code_fence]
+            end
+          end
+
+          # Check for code fence start
+          fence_match = line.match(CODE_FENCE_START)
+          if fence_match
+            token = Token.new(
+              type: :string,
+              start_col: 0,
+              end_col: line.length - 1,
+              text: line
+            )
+            return [[token], :code_fence]
+          end
+
+          # Check line-start patterns
+          line_start_token = check_line_start(line)
+          if line_start_token
+            tokens << line_start_token
+            pos = line_start_token.end_col + 1
+          end
+
+          # Process rest of line with inline patterns
+          while pos < line.length
+            # Skip whitespace
+            if line[pos] =~ /\s/
+              pos += 1
+              next
+            end
+
+            # Try to match a token
+            token = match_token(line, pos)
+            if token
+              tokens << token
+              pos = token.end_col + 1
+            else
+              pos += 1
+            end
+          end
+
+          [tokens, nil]
+        end
+
+        protected
+
+        def compiled_patterns
+          COMPILED_PATTERNS
+        end
+
+        private
+
+        def check_line_start(line)
+          # Heading
+          heading_match = line.match(HEADING_PATTERN)
+          if heading_match
+            level = heading_match[1].length
+            return Token.new(
+              type: :keyword,
+              start_col: 0,
+              end_col: level - 1,
+              text: heading_match[1]
+            )
+          end
+
+          # Horizontal rule
+          if line.match?(HORIZONTAL_RULE_PATTERN)
+            return Token.new(
+              type: :comment,
+              start_col: 0,
+              end_col: line.length - 1,
+              text: line
+            )
+          end
+
+          # Link definition
+          if line.match?(LINK_DEFINITION_PATTERN)
+            return Token.new(
+              type: :constant,
+              start_col: 0,
+              end_col: line.length - 1,
+              text: line
+            )
+          end
+
+          # Blockquote
+          blockquote_match = line.match(BLOCKQUOTE_PATTERN)
+          if blockquote_match
+            return Token.new(
+              type: :comment,
+              start_col: 0,
+              end_col: blockquote_match[0].length - 1,
+              text: blockquote_match[0]
+            )
+          end
+
+          # Unordered list
+          list_match = line.match(UNORDERED_LIST_PATTERN)
+          if list_match
+            return Token.new(
+              type: :operator,
+              start_col: 0,
+              end_col: list_match[0].length - 1,
+              text: list_match[0]
+            )
+          end
+
+          # Ordered list
+          ordered_match = line.match(ORDERED_LIST_PATTERN)
+          if ordered_match
+            return Token.new(
+              type: :number,
+              start_col: 0,
+              end_col: ordered_match[0].length - 1,
+              text: ordered_match[0]
+            )
+          end
+
+          nil
+        end
+
+        def match_token(line, pos)
+          rest = line[pos..]
+
+          COMPILED_PATTERNS.each do |type, pattern|
+            match = rest.match(pattern)
+            next unless match&.begin(0)&.zero?
+
+            return Token.new(
+              type:,
+              start_col: pos,
+              end_col: pos + match[0].length - 1,
+              text: match[0]
+            )
+          end
+
+          nil
+        end
+      end
+    end
+  end
+end

data/lib/mui/syntax/lexers/ruby_lexer.rb

@@ -0,0 +1,114 @@
+# frozen_string_literal: true
+
+module Mui
+  module Syntax
+    module Lexers
+      # Lexer for Ruby source code
+      class RubyLexer < LexerBase
+        KEYWORDS = %w[
+          BEGIN END __ENCODING__ __END__ __FILE__ __LINE__
+          alias and begin break case class def defined? do
+          else elsif end ensure false for if in module next
+          nil not or redo rescue retry return self super then
+          true undef unless until when while yield
+          require require_relative attr_reader attr_writer attr_accessor
+          private protected public include extend prepend
+          raise fail catch throw
+        ].freeze
+
+        # Pre-compiled patterns with \G anchor for position-specific matching
+        # These are compiled once at class load time
+        COMPILED_PATTERNS = [
+          # Single line comment
+          [:comment, /\G#.*/],
+          # Double quoted string (with escape handling)
+          [:string, /\G"(?:[^"\\]|\\.)*"/],
+          # Single quoted string (with escape handling)
+          [:string, /\G'(?:[^'\\]|\\.)*'/],
+          # Symbols
+          [:symbol, /\G:[a-zA-Z_][a-zA-Z0-9_]*[?!]?/],
+          # Float numbers (must be before integer)
+          [:number, /\G\b\d+\.\d+(?:e[+-]?\d+)?\b/i],
+          # Hexadecimal
+          [:number, /\G\b0x[0-9a-f]+\b/i],
+          # Octal
+          [:number, /\G\b0o?[0-7]+\b/],
+          # Binary
+          [:number, /\G\b0b[01]+\b/i],
+          # Integer
+          [:number, /\G\b\d+\b/],
+          # Constants (capitalized identifiers)
+          [:constant, /\G\b[A-Z][a-zA-Z0-9_]*\b/],
+          # Keywords
+          [:keyword, /\G\b(?:BEGIN|END|__ENCODING__|__END__|__FILE__|__LINE__|alias|and|begin|break|case|class|def|defined\?|do|else|elsif|end|ensure|false|for|if|in|module|next|nil|not|or|redo|rescue|retry|return|self|super|then|true|undef|unless|until|when|while|yield|require|require_relative|attr_reader|attr_writer|attr_accessor|private|protected|public|include|extend|prepend|raise|fail|catch|throw)\b/],
+          # Instance variables (@foo, @@foo)
+          [:instance_variable, /\G@{1,2}[a-zA-Z_][a-zA-Z0-9_]*/],
+          # Global variables
+          [:global_variable, /\G\$[a-zA-Z_][a-zA-Z0-9_]*/],
+          # Method calls (.to_i, .each, .map!, .empty?, etc.)
+          [:method_call, /\G\.[a-z_][a-zA-Z0-9_]*[?!]?/],
+          # Identifiers (including method names with ? or !)
+          [:identifier, /\G\b[a-z_][a-zA-Z0-9_]*[?!]?/],
+          # Operators
+          [:operator, %r{\G(?:[+\-*/%&|^~<>=!]+|<<|>>|\*\*)}]
+        ].freeze
+
+        # Multiline patterns (also pre-compiled)
+        BLOCK_COMMENT_END = /\A=end\b/
+        BLOCK_COMMENT_START = /\A=begin\b/
+
+        protected
+
+        # Use pre-compiled class-level patterns
+        def compiled_patterns
+          COMPILED_PATTERNS
+        end
+
+        # Handle =begin...=end block comments
+        def handle_multiline_state(line, pos, state)
+          return [nil, nil, pos] unless state == :block_comment
+
+          # Check for =end
+          text = line[pos..]
+          if BLOCK_COMMENT_END.match?(text)
+            token = Token.new(
+              type: :comment,
+              start_col: pos,
+              end_col: line.length - 1,
+              text:
+            )
+            [token, nil, line.length]
+          else
+            # Entire line is part of block comment
+            unless text.empty?
+              token = Token.new(
+                type: :comment,
+                start_col: pos,
+                end_col: line.length - 1,
+                text:
+              )
+            end
+            [token, :block_comment, line.length]
+          end
+        end
+
+        # Check for =begin block comment start
+        def check_multiline_start(line, pos)
+          return [nil, nil, pos] unless pos.zero?
+
+          if BLOCK_COMMENT_START.match?(line)
+            token = Token.new(
+              type: :comment,
+              start_col: 0,
+              end_col: line.length - 1,
+              text: line
+            )
+            [:block_comment, token, line.length]
+          else
+            [nil, nil, pos]
+          end
+        end
+      end
+    end
+  end
+end