mui 0.1.0 → 0.3.0

data/lib/mui/syntax/language_detector.rb

@@ -0,0 +1,106 @@
+# frozen_string_literal: true
+
+module Mui
+  module Syntax
+    # Detects programming language from file path and provides appropriate lexer
+    class LanguageDetector
+      # Map file extensions to language symbols
+      EXTENSION_MAP = {
+        ".rb" => :ruby,
+        ".ru" => :ruby,
+        ".rake" => :ruby,
+        ".gemspec" => :ruby,
+        ".c" => :c,
+        ".h" => :c,
+        ".y" => :c,
+        ".go" => :go,
+        ".rs" => :rust,
+        ".js" => :javascript,
+        ".mjs" => :javascript,
+        ".cjs" => :javascript,
+        ".jsx" => :javascript,
+        ".ts" => :typescript,
+        ".tsx" => :typescript,
+        ".mts" => :typescript,
+        ".cts" => :typescript,
+        ".md" => :markdown,
+        ".markdown" => :markdown,
+        ".html" => :html,
+        ".htm" => :html,
+        ".xhtml" => :html,
+        ".css" => :css,
+        ".scss" => :css,
+        ".sass" => :css
+      }.freeze
+
+      # Map basenames (files without extension) to language symbols
+      BASENAME_MAP = {
+        "Gemfile" => :ruby,
+        "Rakefile" => :ruby,
+        "Guardfile" => :ruby,
+        "Vagrantfile" => :ruby,
+        "Berksfile" => :ruby,
+        "Capfile" => :ruby,
+        "Thorfile" => :ruby,
+        "Podfile" => :ruby,
+        "Brewfile" => :ruby
+      }.freeze
+
+      class << self
+        # Detect language from file path
+        def detect(file_path)
+          return nil if file_path.nil? || file_path.empty?
+
+          # Try extension first
+          ext = File.extname(file_path).downcase
+          language = EXTENSION_MAP[ext]
+          return language if language
+
+          # Try basename
+          basename = File.basename(file_path)
+          BASENAME_MAP[basename]
+        end
+
+        # Get a lexer instance for a language
+        def lexer_for(language)
+          case language
+          when :ruby
+            Lexers::RubyLexer.new
+          when :c
+            Lexers::CLexer.new
+          when :go
+            Lexers::GoLexer.new
+          when :rust
+            Lexers::RustLexer.new
+          when :javascript
+            Lexers::JavaScriptLexer.new
+          when :typescript
+            Lexers::TypeScriptLexer.new
+          when :markdown
+            Lexers::MarkdownLexer.new
+          when :html
+            Lexers::HtmlLexer.new
+          when :css
+            Lexers::CssLexer.new
+          end
+        end
+
+        # Get a lexer instance for a file path
+        def lexer_for_file(file_path)
+          language = detect(file_path)
+          lexer_for(language)
+        end
+
+        # List all supported languages
+        def supported_languages
+          (EXTENSION_MAP.values + BASENAME_MAP.values).uniq
+        end
+
+        # List all supported extensions
+        def supported_extensions
+          EXTENSION_MAP.keys
+        end
+      end
+    end
+  end
+end

data/lib/mui/syntax/lexer_base.rb

@@ -0,0 +1,106 @@
+# frozen_string_literal: true
+
+module Mui
+  module Syntax
+    # Base class for language-specific lexers
+    # Subclasses should override token_patterns and optionally handle_multiline_state
+    class LexerBase
+      # Tokenize a single line of text
+      # TODO: Refactor to reduce complexity (Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity)
+      def tokenize(line, state = nil)
+        tokens = []
+        pos = 0
+        current_state = state
+
+        while pos < line.length
+          # Handle multiline state first (e.g., inside block comment)
+          if current_state
+            token, new_state, new_pos = handle_multiline_state(line, pos, current_state)
+            if token
+              tokens << token
+              pos = new_pos
+              current_state = new_state
+              next
+            elsif new_state.nil?
+              # State ended, continue normal tokenization
+              current_state = nil
+              pos = new_pos
+              next
+            end
+          end
+
+          # Check for multiline state start
+          new_state, token, new_pos = check_multiline_start(line, pos)
+          if new_state
+            tokens << token if token
+            pos = new_pos
+            current_state = new_state
+            next
+          end
+
+          # Normal token matching
+          token = match_token(line, pos)
+          if token
+            tokens << token
+            pos = token.end_col + 1
+          else
+            # Skip unrecognized character
+            pos += 1
+          end
+        end
+
+        [tokens, current_state]
+      end
+
+      # Check if a state continues to the next line
+      def continuing_state?(state)
+        !state.nil?
+      end
+
+      protected
+
+      # Override in subclass to define token patterns
+      def token_patterns
+        []
+      end
+
+      # Override in subclass to handle multiline constructs
+      def handle_multiline_state(_line, pos, _state)
+        [nil, nil, pos]
+      end
+
+      # Override in subclass to check for multiline construct starts
+      def check_multiline_start(_line, pos)
+        [nil, nil, pos]
+      end
+
+      private
+
+      # Get compiled patterns (cached)
+      # Uses \G anchor for efficient matching at specific position
+      def compiled_patterns
+        @compiled_patterns ||= token_patterns.map do |type, pattern|
+          [type, /\G#{pattern}/]
+        end
+      end
+
+      def match_token(line, pos)
+        return nil if pos >= line.length
+
+        compiled_patterns.each do |type, pattern|
+          match = pattern.match(line, pos)
+          next unless match
+
+          text = match[0]
+          return Token.new(
+            type:,
+            start_col: pos,
+            end_col: pos + text.length - 1,
+            text:
+          )
+        end
+        nil
+      end
+    end
+  end
+end

data/lib/mui/syntax/lexers/c_lexer.rb

@@ -0,0 +1,127 @@
+# frozen_string_literal: true
+
+module Mui
+  module Syntax
+    module Lexers
+      # Lexer for C source code
+      class CLexer < LexerBase
+        # Pre-compiled patterns with \G anchor for position-specific matching
+        # These are compiled once at class load time
+        COMPILED_PATTERNS = [
+          # Single line comment
+          [:comment, %r{\G//.*}],
+          # Single-line block comment /* ... */ on one line
+          [:comment, %r{\G/\*.*?\*/}],
+          # Double quoted string (with escape handling)
+          [:string, /\G"(?:[^"\\]|\\.)*"/],
+          # Character literal
+          [:char, /\G'(?:[^'\\]|\\.)*'/],
+          # Preprocessor directives
+          [:preprocessor, /\G^\s*#\s*(?:include|define|undef|ifdef|ifndef|if|else|elif|endif|error|pragma|line)\b.*/],
+          # Float numbers (must be before integer)
+          [:number, /\G\b\d+\.\d+(?:e[+-]?\d+)?[fFlL]?\b/i],
+          # Hexadecimal
+          [:number, /\G\b0x[0-9a-fA-F]+[uUlL]*\b/],
+          # Octal
+          [:number, /\G\b0[0-7]+[uUlL]*\b/],
+          # Integer
+          [:number, /\G\b\d+[uUlL]*\b/],
+          # Type keywords (int, char, void, etc.)
+          [:type, /\G\b(?:char|double|float|int|long|short|signed|unsigned|void|_Bool|_Complex|_Imaginary)\b/],
+          # Other keywords (if, for, return, const, static, etc.)
+          [:keyword, /\G\b(?:auto|break|case|const|continue|default|do|else|enum|extern|for|goto|if|register|return|sizeof|static|struct|switch|typedef|union|volatile|while|inline|restrict|_Alignas|_Alignof|_Atomic|_Generic|_Noreturn|_Static_assert|_Thread_local)\b/],
+          # Identifiers
+          [:identifier, /\G\b[a-zA-Z_][a-zA-Z0-9_]*\b/],
+          # Operators
+          [:operator, %r{\G(?:[+\-*/%&|^~<>=!]+|->|<<|>>|\+\+|--)}]
+        ].freeze
+
+        # Multiline comment patterns (pre-compiled)
+        BLOCK_COMMENT_END = %r{\*/}
+        BLOCK_COMMENT_START = %r{/\*}
+        BLOCK_COMMENT_START_ANCHOR = %r{\A/\*}
+
+        protected
+
+        # Use pre-compiled class-level patterns
+        def compiled_patterns
+          COMPILED_PATTERNS
+        end
+
+        # Handle /* ... */ block comments that span multiple lines
+        def handle_multiline_state(line, pos, state)
+          return [nil, nil, pos] unless state == :block_comment
+
+          # Look for */
+          end_match = line[pos..].match(BLOCK_COMMENT_END)
+          if end_match
+            end_pos = pos + end_match.begin(0) + 1
+            text = line[pos..end_pos]
+            token = Token.new(
+              type: :comment,
+              start_col: pos,
+              end_col: end_pos,
+              text:
+            )
+            [token, nil, end_pos + 1]
+          else
+            # Entire line is part of block comment
+            text = line[pos..]
+            unless text.empty?
+              token = Token.new(
+                type: :comment,
+                start_col: pos,
+                end_col: line.length - 1,
+                text:
+              )
+            end
+            [token, :block_comment, line.length]
+          end
+        end
+
+        # Check for /* block comment start (that doesn't end on the same line)
+        def check_multiline_start(line, pos)
+          rest = line[pos..]
+
+          # Check for /* that doesn't have a matching */ on this line
+          start_match = rest.match(BLOCK_COMMENT_START)
+          return [nil, nil, pos] unless start_match
+
+          start_pos = pos + start_match.begin(0)
+          after_start = line[(start_pos + 2)..]
+
+          # Check if there's a closing */ on the same line after this /*
+          if after_start&.include?("*/")
+            # There's a closing on this line, let normal token matching handle it
+            [nil, nil, pos]
+          else
+            # No closing on this line, enter block comment state
+            text = line[start_pos..]
+            token = Token.new(
+              type: :comment,
+              start_col: start_pos,
+              end_col: line.length - 1,
+              text:
+            )
+            [:block_comment, token, line.length]
+          end
+        end
+
+        private
+
+        def match_token(line, pos)
+          # First check for start of multiline comment
+          if line[pos..].match?(BLOCK_COMMENT_START_ANCHOR)
+            rest = line[(pos + 2)..]
+            unless rest&.include?("*/")
+              # This will be handled by check_multiline_start
+              return nil
+            end
+          end
+
+          super
+        end
+      end
+    end
+  end
+end

data/lib/mui/syntax/lexers/css_lexer.rb

@@ -0,0 +1,121 @@
+# frozen_string_literal: true
+
+module Mui
+  module Syntax
+    module Lexers
+      # Lexer for CSS source files
+      class CssLexer < LexerBase
+        # Pre-compiled patterns with \G anchor for position-specific matching
+        COMPILED_PATTERNS = [
+          # Single-line block comment /* ... */ on one line
+          [:comment, %r{\G/\*.*?\*/}],
+          # @rules (at-rules)
+          [:preprocessor, /\G@[a-zA-Z-]+/],
+          # Hex color (must be before ID selector - matches 3-8 hex digits only)
+          [:number, /\G#[0-9a-fA-F]{3,8}(?![a-zA-Z0-9_-])/],
+          # ID selector (starts with letter or underscore/hyphen after #)
+          [:constant, /\G#[a-zA-Z_-][a-zA-Z0-9_-]*/],
+          # Class selector
+          [:type, /\G\.[a-zA-Z_-][a-zA-Z0-9_-]*/],
+          # Pseudo-elements and pseudo-classes
+          [:keyword, /\G::?[a-zA-Z-]+(?:\([^)]*\))?/],
+          # Property name (followed by colon)
+          [:identifier, /\G[a-zA-Z-]+(?=\s*:)/],
+          # Double quoted string
+          [:string, /\G"(?:[^"\\]|\\.)*"/],
+          # Single quoted string
+          [:string, /\G'(?:[^'\\]|\\.)*'/],
+          # URL function
+          [:string, /\Gurl\([^)]*\)/i],
+          # Numbers with units
+          [:number, /\G-?\d+\.?\d*(?:px|em|rem|%|vh|vw|vmin|vmax|ch|ex|cm|mm|in|pt|pc|deg|rad|grad|turn|s|ms|Hz|kHz|dpi|dpcm|dppx|fr)?/i],
+          # Functions (calc, rgb, rgba, hsl, var, etc.)
+          [:keyword, /\G[a-zA-Z-]+(?=\()/],
+          # Property values / keywords (important, inherit, etc.)
+          [:constant, /\G!important\b/i],
+          [:constant, /\G\b(?:inherit|initial|unset|revert|none|auto|normal)\b/],
+          # Element selectors and identifiers
+          [:identifier, /\G[a-zA-Z_-][a-zA-Z0-9_-]*/],
+          # Operators and symbols
+          [:operator, /\G[{}():;,>+~*=\[\]]/]
+        ].freeze
+
+        # Multiline comment patterns
+        BLOCK_COMMENT_END = %r{\*/}
+        BLOCK_COMMENT_START = %r{/\*}
+        BLOCK_COMMENT_START_ANCHOR = %r{\A/\*}
+
+        protected
+
+        def compiled_patterns
+          COMPILED_PATTERNS
+        end
+
+        # Handle /* ... */ block comments that span multiple lines
+        def handle_multiline_state(line, pos, state)
+          return [nil, nil, pos] unless state == :block_comment
+
+          end_match = line[pos..].match(BLOCK_COMMENT_END)
+          if end_match
+            end_pos = pos + end_match.begin(0) + 1
+            text = line[pos..end_pos]
+            token = Token.new(
+              type: :comment,
+              start_col: pos,
+              end_col: end_pos,
+              text:
+            )
+            [token, nil, end_pos + 1]
+          else
+            text = line[pos..]
+            token = if text.empty?
+                      nil
+                    else
+                      Token.new(
+                        type: :comment,
+                        start_col: pos,
+                        end_col: line.length - 1,
+                        text:
+                      )
+                    end
+            [token, :block_comment, line.length]
+          end
+        end
+
+        def check_multiline_start(line, pos)
+          rest = line[pos..]
+
+          start_match = rest.match(BLOCK_COMMENT_START)
+          return [nil, nil, pos] unless start_match
+
+          start_pos = pos + start_match.begin(0)
+          after_start = line[(start_pos + 2)..]
+
+          if after_start&.include?("*/")
+            [nil, nil, pos]
+          else
+            text = line[start_pos..]
+            token = Token.new(
+              type: :comment,
+              start_col: start_pos,
+              end_col: line.length - 1,
+              text:
+            )
+            [:block_comment, token, line.length]
+          end
+        end
+
+        private
+
+        def match_token(line, pos)
+          if line[pos..].match?(BLOCK_COMMENT_START_ANCHOR)
+            rest = line[(pos + 2)..]
+            return nil unless rest&.include?("*/")
+          end
+
+          super
+        end
+      end
+    end
+  end
+end

data/lib/mui/syntax/lexers/go_lexer.rb

@@ -0,0 +1,205 @@
+# frozen_string_literal: true
+
+module Mui
+  module Syntax
+    module Lexers
+      # Lexer for Go source code
+      class GoLexer < LexerBase
+        # Go keywords
+        KEYWORDS = %w[
+          break case chan const continue default defer else fallthrough
+          for func go goto if import interface map package range return
+          select struct switch type var
+        ].freeze
+
+        # Go built-in types
+        TYPES = %w[
+          bool byte complex64 complex128 error float32 float64
+          int int8 int16 int32 int64 rune string
+          uint uint8 uint16 uint32 uint64 uintptr
+          any comparable
+        ].freeze
+
+        # Go constants
+        CONSTANTS = %w[true false nil iota].freeze
+
+        # Pre-compiled patterns with \G anchor for position-specific matching
+        COMPILED_PATTERNS = [
+          # Single line comment
+          [:comment, %r{\G//.*}],
+          # Single-line block comment /* ... */ on one line
+          [:comment, %r{\G/\*.*?\*/}],
+          # Raw string literal (backtick)
+          [:string, /\G`[^`]*`/],
+          # Double quoted string (with escape handling)
+          [:string, /\G"(?:[^"\\]|\\.)*"/],
+          # Character literal (rune)
+          [:char, /\G'(?:[^'\\]|\\.)*'/],
+          # Float numbers (must be before integer)
+          [:number, /\G\b\d+\.\d+(?:e[+-]?\d+)?\b/i],
+          # Hexadecimal
+          [:number, /\G\b0x[0-9a-fA-F]+\b/i],
+          # Octal
+          [:number, /\G\b0o[0-7]+\b/i],
+          # Binary
+          [:number, /\G\b0b[01]+\b/i],
+          # Integer
+          [:number, /\G\b\d+\b/],
+          # Constants (true, false, nil, iota)
+          [:constant, /\G\b(?:true|false|nil|iota)\b/],
+          # Types
+          [:type, /\G\b(?:bool|byte|complex64|complex128|error|float32|float64|int|int8|int16|int32|int64|rune|string|uint|uint8|uint16|uint32|uint64|uintptr|any|comparable)\b/],
+          # Keywords
+          [:keyword, /\G\b(?:break|case|chan|const|continue|default|defer|else|fallthrough|for|func|go|goto|if|import|interface|map|package|range|return|select|struct|switch|type|var)\b/],
+          # Exported identifiers (start with uppercase)
+          [:constant, /\G\b[A-Z][a-zA-Z0-9_]*\b/],
+          # Regular identifiers
+          [:identifier, /\G\b[a-z_][a-zA-Z0-9_]*\b/],
+          # Operators
+          [:operator, %r{\G(?:&&|\|\||<-|<<=?|>>=?|&\^=?|[+\-*/%&|^<>=!]=?|:=|\+\+|--)}]
+        ].freeze
+
+        # Multiline comment patterns (pre-compiled)
+        BLOCK_COMMENT_END = %r{\*/}
+        BLOCK_COMMENT_START = %r{/\*}
+        BLOCK_COMMENT_START_ANCHOR = %r{\A/\*}
+
+        # Raw string patterns (pre-compiled)
+        RAW_STRING_START = /\A`/
+        RAW_STRING_END = /`/
+
+        protected
+
+        def compiled_patterns
+          COMPILED_PATTERNS
+        end
+
+        # Handle /* ... */ block comments and raw strings that span multiple lines
+        def handle_multiline_state(line, pos, state)
+          case state
+          when :block_comment
+            handle_block_comment(line, pos)
+          when :raw_string
+            handle_raw_string(line, pos)
+          else
+            [nil, nil, pos]
+          end
+        end
+
+        def check_multiline_start(line, pos)
+          rest = line[pos..]
+
+          # Check for raw string start
+          if rest.match?(RAW_STRING_START)
+            after_start = line[(pos + 1)..]
+            unless after_start&.include?("`")
+              # No closing on this line, enter raw string state
+              text = line[pos..]
+              token = Token.new(
+                type: :string,
+                start_col: pos,
+                end_col: line.length - 1,
+                text:
+              )
+              return [:raw_string, token, line.length]
+            end
+          end
+
+          # Check for /* that doesn't have a matching */ on this line
+          start_match = rest.match(BLOCK_COMMENT_START)
+          return [nil, nil, pos] unless start_match
+
+          start_pos = pos + start_match.begin(0)
+          after_start = line[(start_pos + 2)..]
+
+          if after_start&.include?("*/")
+            [nil, nil, pos]
+          else
+            text = line[start_pos..]
+            token = Token.new(
+              type: :comment,
+              start_col: start_pos,
+              end_col: line.length - 1,
+              text:
+            )
+            [:block_comment, token, line.length]
+          end
+        end
+
+        private
+
+        def handle_block_comment(line, pos)
+          end_match = line[pos..].match(BLOCK_COMMENT_END)
+          if end_match
+            end_pos = pos + end_match.begin(0) + 1
+            text = line[pos..end_pos]
+            token = Token.new(
+              type: :comment,
+              start_col: pos,
+              end_col: end_pos,
+              text:
+            )
+            [token, nil, end_pos + 1]
+          else
+            text = line[pos..]
+            token = if text.empty?
+                      nil
+                    else
+                      Token.new(
+                        type: :comment,
+                        start_col: pos,
+                        end_col: line.length - 1,
+                        text:
+                      )
+                    end
+            [token, :block_comment, line.length]
+          end
+        end
+
+        def handle_raw_string(line, pos)
+          end_match = line[pos..].match(RAW_STRING_END)
+          if end_match
+            end_pos = pos + end_match.begin(0)
+            text = line[pos..end_pos]
+            token = Token.new(
+              type: :string,
+              start_col: pos,
+              end_col: end_pos,
+              text:
+            )
+            [token, nil, end_pos + 1]
+          else
+            text = line[pos..]
+            token = if text.empty?
+                      nil
+                    else
+                      Token.new(
+                        type: :string,
+                        start_col: pos,
+                        end_col: line.length - 1,
+                        text:
+                      )
+                    end
+            [token, :raw_string, line.length]
+          end
+        end
+
+        def match_token(line, pos)
+          # Check for start of raw string
+          if line[pos..].match?(RAW_STRING_START)
+            rest = line[(pos + 1)..]
+            return nil unless rest&.include?("`")
+          end
+
+          # Check for start of multiline comment
+          if line[pos..].match?(BLOCK_COMMENT_START_ANCHOR)
+            rest = line[(pos + 2)..]
+            return nil unless rest&.include?("*/")
+          end
+
+          super
+        end
+      end
+    end
+  end
+end