mui 0.2.0 → 0.4.0

data/lib/mui/syntax/lexers/html_lexer.rb

@@ -0,0 +1,118 @@
+# frozen_string_literal: true
+
+module Mui
+  module Syntax
+    module Lexers
+      # Lexer for HTML source files
+      class HtmlLexer < LexerBase
+        # Pre-compiled patterns with \G anchor for position-specific matching
+        COMPILED_PATTERNS = [
+          # HTML comment (single line)
+          [:comment, /\G<!--.*?-->/],
+          # DOCTYPE declaration
+          [:preprocessor, /\G<!DOCTYPE[^>]*>/i],
+          # CDATA section
+          [:string, /\G<!\[CDATA\[.*?\]\]>/],
+          # Closing tag
+          [:keyword, %r{\G</[a-zA-Z][a-zA-Z0-9-]*\s*>}],
+          # Self-closing tag
+          [:keyword, %r{\G<[a-zA-Z][a-zA-Z0-9-]*(?:\s+[^>]*)?/>}],
+          # Opening tag with attributes
+          [:keyword, /\G<[a-zA-Z][a-zA-Z0-9-]*(?=[\s>])/],
+          # Tag closing bracket
+          [:keyword, /\G>/],
+          # Attribute name
+          [:type, /\G[a-zA-Z][a-zA-Z0-9_-]*(?==)/],
+          # Double quoted attribute value
+          [:string, /\G"[^"]*"/],
+          # Single quoted attribute value
+          [:string, /\G'[^']*'/],
+          # Unquoted attribute value (limited characters)
+          [:string, /\G=[^\s>"']+/],
+          # HTML entities
+          [:constant, /\G&(?:#\d+|#x[0-9a-fA-F]+|[a-zA-Z]+);/],
+          # Equal sign (for attributes)
+          [:operator, /\G=/]
+        ].freeze
+
+        # Multiline comment patterns
+        COMMENT_START = /<!--/
+        COMMENT_END = /-->/
+        COMMENT_START_ANCHOR = /\G<!--/
+
+        protected
+
+        def compiled_patterns
+          COMPILED_PATTERNS
+        end
+
+        # Handle multiline HTML comments
+        def handle_multiline_state(line, pos, state)
+          return [nil, nil, pos] unless state == :html_comment
+
+          end_match = line[pos..].match(COMMENT_END)
+          if end_match
+            end_pos = pos + end_match.begin(0) + 2 # --> is 3 chars
+            text = line[pos..end_pos]
+            token = Token.new(
+              type: :comment,
+              start_col: pos,
+              end_col: end_pos,
+              text:
+            )
+            [token, nil, end_pos + 1]
+          else
+            text = line[pos..]
+            token = if text.empty?
+                      nil
+                    else
+                      Token.new(
+                        type: :comment,
+                        start_col: pos,
+                        end_col: line.length - 1,
+                        text:
+                      )
+                    end
+            [token, :html_comment, line.length]
+          end
+        end
+
+        def check_multiline_start(line, pos)
+          rest = line[pos..]
+
+          # Check for <!-- that doesn't have --> on this line
+          start_match = rest.match(COMMENT_START)
+          return [nil, nil, pos] unless start_match
+
+          start_pos = pos + start_match.begin(0)
+          after_start = line[(start_pos + 4)..] # Skip <!--
+
+          if after_start&.include?("-->")
+            [nil, nil, pos]
+          else
+            text = line[start_pos..]
+            token = Token.new(
+              type: :comment,
+              start_col: start_pos,
+              end_col: line.length - 1,
+              text:
+            )
+            [:html_comment, token, line.length]
+          end
+        end
+
+        private
+
+        def match_token(line, pos)
+          # Check for start of multiline comment
+          if line[pos..].match?(COMMENT_START_ANCHOR)
+            rest = line[(pos + 4)..]
+            return nil unless rest&.include?("-->")
+          end
+
+          super
+        end
+      end
+    end
+  end
+end

data/lib/mui/syntax/lexers/javascript_lexer.rb

@@ -0,0 +1,219 @@
+# frozen_string_literal: true
+
+module Mui
+  module Syntax
+    module Lexers
+      # Lexer for JavaScript source code
+      class JavaScriptLexer < LexerBase
+        # JavaScript keywords
+        KEYWORDS = %w[
+          async await break case catch class const continue debugger default
+          delete do else export extends finally for function if import in
+          instanceof let new return static switch throw try typeof var void
+          while with yield
+        ].freeze
+
+        # JavaScript built-in types and values
+        CONSTANTS = %w[true false null undefined NaN Infinity this super].freeze
+
+        # Pre-compiled patterns with \G anchor for position-specific matching
+        COMPILED_PATTERNS = [
+          # Single line comment
+          [:comment, %r{\G//.*}],
+          # Single-line block comment /* ... */ on one line
+          [:comment, %r{\G/\*.*?\*/}],
+          # Template literal (single line)
+          [:string, /\G`[^`]*`/],
+          # Double quoted string (with escape handling)
+          [:string, /\G"(?:[^"\\]|\\.)*"/],
+          # Single quoted string (with escape handling)
+          [:string, /\G'(?:[^'\\]|\\.)*'/],
+          # Regular expression literal
+          [:regex, %r{\G/(?:[^/\\]|\\.)+/[gimsuy]*}],
+          # Float numbers (must be before integer)
+          [:number, /\G\b\d+\.\d+(?:e[+-]?\d+)?\b/i],
+          # Hexadecimal
+          [:number, /\G\b0x[0-9a-fA-F]+n?\b/i],
+          # Octal
+          [:number, /\G\b0o[0-7]+n?\b/i],
+          # Binary
+          [:number, /\G\b0b[01]+n?\b/i],
+          # Integer (with optional BigInt suffix)
+          [:number, /\G\b\d+n?\b/],
+          # Constants (true, false, null, undefined, NaN, Infinity, this, super)
+          [:constant, /\G\b(?:true|false|null|undefined|NaN|Infinity|this|super)\b/],
+          # Keywords
+          [:keyword, /\G\b(?:async|await|break|case|catch|class|const|continue|debugger|default|delete|do|else|export|extends|finally|for|function|if|import|in|instanceof|let|new|return|static|switch|throw|try|typeof|var|void|while|with|yield)\b/],
+          # Class/constructor names (start with uppercase)
+          [:constant, /\G\b[A-Z][a-zA-Z0-9_]*\b/],
+          # Regular identifiers
+          [:identifier, /\G\b[a-zA-Z_$][a-zA-Z0-9_$]*\b/],
+          # Operators (=>must come before = patterns)
+          [:operator, %r{\G(?:\.{3}|=>|&&=?|\|\|=?|\?\?=?|===?|!==?|>>>?=?|<<=?|\+\+|--|\?\.?|[+\-*/%&|^<>=!]=?)}]
+        ].freeze
+
+        # Multiline patterns (pre-compiled)
+        BLOCK_COMMENT_END = %r{\*/}
+        BLOCK_COMMENT_START = %r{/\*}
+        BLOCK_COMMENT_START_ANCHOR = %r{\A/\*}
+
+        # Template literal patterns
+        TEMPLATE_LITERAL_START = /\A`/
+        TEMPLATE_LITERAL_END = /`/
+
+        # Override tokenize to post-process function definitions
+        def tokenize(line, state = nil)
+          tokens, new_state = super
+
+          # Convert identifiers after 'function' keyword to function_definition
+          tokens.each_with_index do |token, i|
+            next unless i.positive? &&
+                        tokens[i - 1].type == :keyword &&
+                        tokens[i - 1].text == "function" &&
+                        token.type == :identifier
+
+            tokens[i] = Token.new(
+              type: :function_definition,
+              start_col: token.start_col,
+              end_col: token.end_col,
+              text: token.text
+            )
+          end
+
+          [tokens, new_state]
+        end
+
+        protected
+
+        def compiled_patterns
+          COMPILED_PATTERNS
+        end
+
+        # Handle multiline constructs
+        def handle_multiline_state(line, pos, state)
+          case state
+          when :block_comment
+            handle_block_comment(line, pos)
+          when :template_literal
+            handle_template_literal(line, pos)
+          else
+            [nil, nil, pos]
+          end
+        end
+
+        def check_multiline_start(line, pos)
+          rest = line[pos..]
+
+          # Check for template literal start
+          if rest.match?(TEMPLATE_LITERAL_START)
+            after_start = line[(pos + 1)..]
+            unless after_start&.include?("`")
+              text = line[pos..]
+              token = Token.new(
+                type: :string,
+                start_col: pos,
+                end_col: line.length - 1,
+                text:
+              )
+              return [:template_literal, token, line.length]
+            end
+          end
+
+          # Check for /* that doesn't have a matching */ on this line
+          start_match = rest.match(BLOCK_COMMENT_START)
+          return [nil, nil, pos] unless start_match
+
+          start_pos = pos + start_match.begin(0)
+          after_start = line[(start_pos + 2)..]
+
+          if after_start&.include?("*/")
+            [nil, nil, pos]
+          else
+            text = line[start_pos..]
+            token = Token.new(
+              type: :comment,
+              start_col: start_pos,
+              end_col: line.length - 1,
+              text:
+            )
+            [:block_comment, token, line.length]
+          end
+        end
+
+        private
+
+        def handle_block_comment(line, pos)
+          end_match = line[pos..].match(BLOCK_COMMENT_END)
+          if end_match
+            end_pos = pos + end_match.begin(0) + 1
+            text = line[pos..end_pos]
+            token = Token.new(
+              type: :comment,
+              start_col: pos,
+              end_col: end_pos,
+              text:
+            )
+            [token, nil, end_pos + 1]
+          else
+            text = line[pos..]
+            token = if text.empty?
+                      nil
+                    else
+                      Token.new(
+                        type: :comment,
+                        start_col: pos,
+                        end_col: line.length - 1,
+                        text:
+                      )
+                    end
+            [token, :block_comment, line.length]
+          end
+        end
+
+        def handle_template_literal(line, pos)
+          end_match = line[pos..].match(TEMPLATE_LITERAL_END)
+          if end_match
+            end_pos = pos + end_match.begin(0)
+            text = line[pos..end_pos]
+            token = Token.new(
+              type: :string,
+              start_col: pos,
+              end_col: end_pos,
+              text:
+            )
+            [token, nil, end_pos + 1]
+          else
+            text = line[pos..]
+            token = if text.empty?
+                      nil
+                    else
+                      Token.new(
+                        type: :string,
+                        start_col: pos,
+                        end_col: line.length - 1,
+                        text:
+                      )
+                    end
+            [token, :template_literal, line.length]
+          end
+        end
+
+        def match_token(line, pos)
+          # Check for start of template literal
+          if line[pos..].match?(TEMPLATE_LITERAL_START)
+            rest = line[(pos + 1)..]
+            return nil unless rest&.include?("`")
+          end
+
+          # Check for start of multiline comment
+          if line[pos..].match?(BLOCK_COMMENT_START_ANCHOR)
+            rest = line[(pos + 2)..]
+            return nil unless rest&.include?("*/")
+          end
+
+          super
+        end
+      end
+    end
+  end
+end

data/lib/mui/syntax/lexers/markdown_lexer.rb

@@ -0,0 +1,210 @@
+# frozen_string_literal: true
+
+module Mui
+  module Syntax
+    module Lexers
+      # Lexer for Markdown source files
+      class MarkdownLexer < LexerBase
+        # Pre-compiled patterns with \G anchor for position-specific matching
+        # Markdown is line-oriented, so we check line-start patterns separately
+        COMPILED_PATTERNS = [
+          # Inline code (backtick)
+          [:string, /\G`[^`]+`/],
+          # Bold with asterisks
+          [:keyword, /\G\*\*[^*]+\*\*/],
+          # Bold with underscores
+          [:keyword, /\G__[^_]+__/],
+          # Italic with asterisks
+          [:comment, /\G\*[^*]+\*/],
+          # Italic with underscores
+          [:comment, /\G_[^_]+_/],
+          # Strikethrough
+          [:comment, /\G~~[^~]+~~/],
+          # Link [text](url)
+          [:constant, /\G\[[^\]]+\]\([^)]+\)/],
+          # Image ![alt](url)
+          [:constant, /\G!\[[^\]]*\]\([^)]+\)/],
+          # Reference link [text][ref]
+          [:constant, /\G\[[^\]]+\]\[[^\]]*\]/],
+          # Autolink <url> or <email>
+          [:constant, /\G<[a-zA-Z][a-zA-Z0-9+.-]*:[^>]+>/],
+          # HTML tags
+          [:preprocessor, %r{\G</?[a-zA-Z][a-zA-Z0-9]*[^>]*>}]
+        ].freeze
+
+        # Line-start patterns (checked at beginning of line)
+        HEADING_PATTERN = /\A(\#{1,6})\s+(.*)$/
+        BLOCKQUOTE_PATTERN = /\A>\s*/
+        UNORDERED_LIST_PATTERN = /\A\s*[-*+]\s+/
+        ORDERED_LIST_PATTERN = /\A\s*\d+\.\s+/
+        HORIZONTAL_RULE_PATTERN = /\A([-*_])\s*\1\s*\1[\s\1]*$/
+        CODE_FENCE_START = /\A```(\w*)/
+        CODE_FENCE_END = /\A```\s*$/
+        LINK_DEFINITION_PATTERN = /\A\s*\[[^\]]+\]:\s+\S+/
+
+        # Override tokenize to handle line-start patterns
+        def tokenize(line, state = nil)
+          tokens = []
+          pos = 0
+
+          # Handle code fence state
+          if state == :code_fence
+            if line.match?(CODE_FENCE_END)
+              token = Token.new(
+                type: :string,
+                start_col: 0,
+                end_col: line.length - 1,
+                text: line
+              )
+              return [[token], nil]
+            else
+              unless line.empty?
+                token = Token.new(
+                  type: :string,
+                  start_col: 0,
+                  end_col: line.length - 1,
+                  text: line
+                )
+              end
+              return [token ? [token] : [], :code_fence]
+            end
+          end
+
+          # Check for code fence start
+          fence_match = line.match(CODE_FENCE_START)
+          if fence_match
+            token = Token.new(
+              type: :string,
+              start_col: 0,
+              end_col: line.length - 1,
+              text: line
+            )
+            return [[token], :code_fence]
+          end
+
+          # Check line-start patterns
+          line_start_token = check_line_start(line)
+          if line_start_token
+            tokens << line_start_token
+            pos = line_start_token.end_col + 1
+          end
+
+          # Process rest of line with inline patterns
+          while pos < line.length
+            # Skip whitespace
+            if line[pos] =~ /\s/
+              pos += 1
+              next
+            end
+
+            # Try to match a token
+            token = match_token(line, pos)
+            if token
+              tokens << token
+              pos = token.end_col + 1
+            else
+              pos += 1
+            end
+          end
+
+          [tokens, nil]
+        end
+
+        protected
+
+        def compiled_patterns
+          COMPILED_PATTERNS
+        end
+
+        private
+
+        def check_line_start(line)
+          # Heading
+          heading_match = line.match(HEADING_PATTERN)
+          if heading_match
+            level = heading_match[1].length
+            return Token.new(
+              type: :keyword,
+              start_col: 0,
+              end_col: level - 1,
+              text: heading_match[1]
+            )
+          end
+
+          # Horizontal rule
+          if line.match?(HORIZONTAL_RULE_PATTERN)
+            return Token.new(
+              type: :comment,
+              start_col: 0,
+              end_col: line.length - 1,
+              text: line
+            )
+          end
+
+          # Link definition
+          if line.match?(LINK_DEFINITION_PATTERN)
+            return Token.new(
+              type: :constant,
+              start_col: 0,
+              end_col: line.length - 1,
+              text: line
+            )
+          end
+
+          # Blockquote
+          blockquote_match = line.match(BLOCKQUOTE_PATTERN)
+          if blockquote_match
+            return Token.new(
+              type: :comment,
+              start_col: 0,
+              end_col: blockquote_match[0].length - 1,
+              text: blockquote_match[0]
+            )
+          end
+
+          # Unordered list
+          list_match = line.match(UNORDERED_LIST_PATTERN)
+          if list_match
+            return Token.new(
+              type: :operator,
+              start_col: 0,
+              end_col: list_match[0].length - 1,
+              text: list_match[0]
+            )
+          end
+
+          # Ordered list
+          ordered_match = line.match(ORDERED_LIST_PATTERN)
+          if ordered_match
+            return Token.new(
+              type: :number,
+              start_col: 0,
+              end_col: ordered_match[0].length - 1,
+              text: ordered_match[0]
+            )
+          end
+
+          nil
+        end
+
+        def match_token(line, pos)
+          rest = line[pos..]
+
+          COMPILED_PATTERNS.each do |type, pattern|
+            match = rest.match(pattern)
+            next unless match&.begin(0)&.zero?
+
+            return Token.new(
+              type:,
+              start_col: pos,
+              end_col: pos + match[0].length - 1,
+              text: match[0]
+            )
+          end
+
+          nil
+        end
+      end
+    end
+  end
+end

data/lib/mui/syntax/lexers/ruby_lexer.rb

@@ -47,6 +47,9 @@ module Mui
           [:global_variable, /\G\$[a-zA-Z_][a-zA-Z0-9_]*/],
           # Method calls (.to_i, .each, .map!, .empty?, etc.)
           [:method_call, /\G\.[a-z_][a-zA-Z0-9_]*[?!]?/],
+          # Function/method definition names (def の後)
+          # Note: def self.method_name is handled by method_call pattern (same color)
+          [:function_definition, /\G(?<=def )[a-z_][a-zA-Z0-9_]*[?!=]?/],
           # Identifiers (including method names with ? or !)
           [:identifier, /\G\b[a-z_][a-zA-Z0-9_]*[?!]?/],
           # Operators

data/lib/mui/syntax/lexers/rust_lexer.rb

@@ -0,0 +1,150 @@
+# frozen_string_literal: true
+
+module Mui
+  module Syntax
+    module Lexers
+      # Lexer for Rust source code
+      class RustLexer < LexerBase
+        # Rust keywords
+        KEYWORDS = %w[
+          as async await break const continue crate dyn else enum extern
+          false fn for if impl in let loop match mod move mut pub ref
+          return self Self static struct super trait true type unsafe use
+          where while
+        ].freeze
+
+        # Rust primitive types
+        TYPES = %w[
+          bool char str
+          i8 i16 i32 i64 i128 isize
+          u8 u16 u32 u64 u128 usize
+          f32 f64
+        ].freeze
+
+        # Pre-compiled patterns with \G anchor for position-specific matching
+        COMPILED_PATTERNS = [
+          # Doc comments (must be before regular comments)
+          [:comment, %r{\G///.*}],
+          [:comment, %r{\G//!.*}],
+          # Single line comment
+          [:comment, %r{\G//.*}],
+          # Single-line block comment /* ... */ on one line
+          [:comment, %r{\G/\*.*?\*/}],
+          # Attributes
+          [:preprocessor, /\G#!\[[^\]]*\]/],
+          [:preprocessor, /\G#\[[^\]]*\]/],
+          # Character literal - single char or escape sequence (must be before lifetime)
+          [:char, /\G'(?:[^'\\]|\\.)'/],
+          # Lifetime
+          [:symbol, /\G'[a-z_][a-zA-Z0-9_]*/],
+          # Raw string r#"..."#
+          [:string, /\Gr#+"[^"]*"#+/],
+          # Byte string
+          [:string, /\Gb"(?:[^"\\]|\\.)*"/],
+          # Regular string
+          [:string, /\G"(?:[^"\\]|\\.)*"/],
+          # Float numbers (must be before integer)
+          [:number, /\G\b\d+\.\d+(?:e[+-]?\d+)?(?:f32|f64)?\b/i],
+          # Hexadecimal
+          [:number, /\G\b0x[0-9a-fA-F_]+(?:i8|i16|i32|i64|i128|isize|u8|u16|u32|u64|u128|usize)?\b/],
+          # Octal
+          [:number, /\G\b0o[0-7_]+(?:i8|i16|i32|i64|i128|isize|u8|u16|u32|u64|u128|usize)?\b/],
+          # Binary
+          [:number, /\G\b0b[01_]+(?:i8|i16|i32|i64|i128|isize|u8|u16|u32|u64|u128|usize)?\b/],
+          # Integer
+          [:number, /\G\b\d[0-9_]*(?:i8|i16|i32|i64|i128|isize|u8|u16|u32|u64|u128|usize)?\b/],
+          # Macros (identifier followed by !)
+          [:macro, /\G\b[a-z_][a-zA-Z0-9_]*!/],
+          # Primitive types
+          [:type, /\G\b(?:bool|char|str|i8|i16|i32|i64|i128|isize|u8|u16|u32|u64|u128|usize|f32|f64)\b/],
+          # Keywords
+          [:keyword, /\G\b(?:as|async|await|break|const|continue|crate|dyn|else|enum|extern|false|fn|for|if|impl|in|let|loop|match|mod|move|mut|pub|ref|return|self|Self|static|struct|super|trait|true|type|unsafe|use|where|while)\b/],
+          # Function definition names (fn の後)
+          [:function_definition, /\G(?<=fn )[a-z_][a-zA-Z0-9_]*/],
+          # Type names (start with uppercase)
+          [:constant, /\G\b[A-Z][a-zA-Z0-9_]*\b/],
+          # Regular identifiers
+          [:identifier, /\G\b[a-z_][a-zA-Z0-9_]*\b/],
+          # Operators
+          [:operator, %r{\G(?:&&|\|\||<<|>>|=>|->|::|\.\.=?|[+\-*/%&|^<>=!]=?|\?)}]
+        ].freeze
+
+        # Multiline comment patterns (pre-compiled)
+        BLOCK_COMMENT_END = %r{\*/}
+        BLOCK_COMMENT_START = %r{/\*}
+        BLOCK_COMMENT_START_ANCHOR = %r{\A/\*}
+
+        protected
+
+        def compiled_patterns
+          COMPILED_PATTERNS
+        end
+
+        # Handle /* ... */ block comments that span multiple lines
+        def handle_multiline_state(line, pos, state)
+          return [nil, nil, pos] unless state == :block_comment
+
+          end_match = line[pos..].match(BLOCK_COMMENT_END)
+          if end_match
+            end_pos = pos + end_match.begin(0) + 1
+            text = line[pos..end_pos]
+            token = Token.new(
+              type: :comment,
+              start_col: pos,
+              end_col: end_pos,
+              text:
+            )
+            [token, nil, end_pos + 1]
+          else
+            text = line[pos..]
+            token = if text.empty?
+                      nil
+                    else
+                      Token.new(
+                        type: :comment,
+                        start_col: pos,
+                        end_col: line.length - 1,
+                        text:
+                      )
+                    end
+            [token, :block_comment, line.length]
+          end
+        end
+
+        def check_multiline_start(line, pos)
+          rest = line[pos..]
+
+          start_match = rest.match(BLOCK_COMMENT_START)
+          return [nil, nil, pos] unless start_match
+
+          start_pos = pos + start_match.begin(0)
+          after_start = line[(start_pos + 2)..]
+
+          if after_start&.include?("*/")
+            [nil, nil, pos]
+          else
+            text = line[start_pos..]
+            token = Token.new(
+              type: :comment,
+              start_col: start_pos,
+              end_col: line.length - 1,
+              text:
+            )
+            [:block_comment, token, line.length]
+          end
+        end
+
+        private
+
+        def match_token(line, pos)
+          if line[pos..].match?(BLOCK_COMMENT_START_ANCHOR)
+            rest = line[(pos + 2)..]
+            return nil unless rest&.include?("*/")
+          end
+
+          super
+        end
+      end
+    end
+  end
+end