lexer_kit 0.5.0

data/lib/lexer_kit/cli/commands.rb

@@ -0,0 +1,389 @@
+# frozen_string_literal: true
+
+module LexerKit
+  module CLI
+    module Commands
+      # Compile DSL to .lkt1 or .lkb1 binary format
+      module Compile
+        def self.run(argv)
+          options = { output: nil, verbose: false, dry_run: false }
+
+          parser = OptionParser.new do |opts|
+            opts.banner = "Usage: lexer_kit compile [options] <file.rb>"
+
+            opts.on("-o", "--output FILE", "Output file (default: <input>.lkt1)") do |v|
+              options[:output] = v
+            end
+
+            opts.on("--verbose", "Show conflict warnings") do
+              options[:verbose] = true
+            end
+
+            opts.on("-n", "--dry-run", "Check for conflicts without generating output") do
+              options[:dry_run] = true
+            end
+
+            opts.on("-h", "--help", "Show this help") do
+              puts opts
+              return 0
+            end
+          end
+
+          parser.parse!(argv)
+
+          if argv.empty?
+            warn "error: No input file specified"
+            warn parser.banner
+            return 1
+          end
+
+          path = argv.shift
+          unless File.exist?(path)
+            warn "error: File not found: #{path}"
+            return 1
+          end
+
+          # Load builder
+          builder = LexerKit.load_builder(path)
+
+          # Check for conflicts with --verbose or --dry-run
+          if options[:verbose] || options[:dry_run]
+            conflicts = builder.check_conflicts
+            if conflicts.any?
+              conflicts.each do |c|
+                warn "warning: #{c.token1} vs #{c.token2}: #{c.description}"
+              end
+              warn ""
+              warn "#{conflicts.size} potential conflict(s) found"
+            elsif options[:dry_run]
+              warn "No conflicts found."
+            end
+          end
+
+          # Exit early if dry-run
+          if options[:dry_run]
+            return 0
+          end
+
+          # Compile
+          compiled = builder.compile
+
+          # Write output
+          output_path = options[:output] || path.sub(/\.rb$/, ".lkt1")
+          if output_path.end_with?(".lkb1")
+            Format::LKB1.save(compiled, path: output_path)
+          else
+            Format::LKT1.save(compiled, path: output_path)
+          end
+          warn "Compiled: #{output_path}"
+
+          0
+        end
+      end
+
+      # Show lexer information
+      module Info
+        def self.run(argv)
+          parser = OptionParser.new do |opts|
+            opts.banner = "Usage: lexer_kit info <file>"
+
+            opts.on("-h", "--help", "Show this help") do
+              puts opts
+              return 0
+            end
+          end
+
+          parser.parse!(argv)
+
+          if argv.empty?
+            warn "error: No input file specified"
+            warn parser.banner
+            return 1
+          end
+
+          path = argv.shift
+          lexer = CLI.load_lexer(path)
+
+          puts "LexerKit Compiled Program"
+          puts "  Version:      #{lexer.version}"
+          puts "  Native:       #{LexerKit.native? ? 'enabled' : 'disabled'}"
+          puts
+
+          puts "Tokens (#{lexer.tokens.size}):"
+          lexer.tokens.each_with_index do |name, id|
+            puts "  #{id.to_s.rjust(3)}: #{name}"
+          end
+          puts
+
+          puts "Modes (#{lexer.modes.size}):"
+          lexer.modes.each do |name|
+            offset = lexer.mode_offset(name)
+            puts "  - #{name} (offset: #{offset})"
+          end
+          puts
+
+          puts "Statistics:"
+          puts "  Instructions: #{lexer.instructions.size}"
+          puts "  DFA tables:   #{lexer.dfa_tables.size}"
+          puts "  Jump tables:  #{lexer.jump_tables.size}"
+          puts "  Keywords:     #{lexer.keyword_tables.size}"
+          puts "  Constants:    #{lexer.constant_pool.size}"
+
+          # Calculate binary size
+          binary_size = lexer.to_binary.bytesize
+          puts "  Binary size:  #{format_size(binary_size)}"
+
+          0
+        end
+
+        def self.format_size(bytes)
+          if bytes < 1024
+            "#{bytes} bytes"
+          elsif bytes < 1024 * 1024
+            "#{(bytes / 1024.0).round(1)} KB"
+          else
+            "#{(bytes / 1024.0 / 1024.0).round(2)} MB"
+          end
+        end
+      end
+
+      # Tokenize input
+      module Lex
+        def self.run(argv)
+          options = { format: "table", color: $stdout.tty? }
+
+          parser = OptionParser.new do |opts|
+            opts.banner = "Usage: lexer_kit lex <lexer> [input]"
+
+            opts.on("-f", "--format FORMAT", %w[table json simple], "Output format (table, json, simple)") do |v|
+              options[:format] = v
+            end
+
+            opts.on("--no-color", "Disable colored output") do
+              options[:color] = false
+            end
+
+            opts.on("-h", "--help", "Show this help") do
+              puts opts
+              return 0
+            end
+          end
+
+          parser.parse!(argv)
+
+          if argv.empty?
+            warn "error: No lexer file specified"
+            warn parser.banner
+            return 1
+          end
+
+          lexer_path = argv.shift
+          lexer = CLI.load_lexer(lexer_path)
+
+          # Read input
+          input = if argv.empty?
+                    $stdin.read
+                  else
+                    File.read(argv.shift)
+                  end
+
+          # Collect tokens
+          tokens = []
+          source = Core::Source.new(input)
+          source.line_index!
+
+          lexer.lowlevel_each(input) do |tok_id, start, len|
+            line, col = source.line_col(start)
+            tokens << {
+              id: tok_id,
+              name: lexer.token_name(tok_id),
+              text: input.byteslice(start, len),
+              line: line,
+              col: col,
+              start: start,
+              len: len
+            }
+          end
+
+          # Output
+          case options[:format]
+          when "json"
+            output_json(tokens)
+          when "simple"
+            output_simple(tokens)
+          else
+            output_table(tokens, options[:color])
+          end
+
+          0
+        end
+
+        def self.output_json(tokens)
+          result = tokens.map do |t|
+            {
+              token: t[:name].to_s,
+              text: t[:text],
+              line: t[:line],
+              col: t[:col],
+              start: t[:start],
+              len: t[:len]
+            }
+          end
+          puts JSON.pretty_generate(result)
+        end
+
+        def self.output_simple(tokens)
+          tokens.each do |t|
+            puts "#{t[:name]} #{t[:text].inspect}"
+          end
+        end
+
+        def self.output_table(tokens, color)
+          # Header
+          puts "  LINE:COL  TOKEN            TEXT"
+          puts "  --------  ---------------  --------------------"
+
+          tokens.each do |t|
+            pos = "#{t[:line]}:#{t[:col]}"
+            name = t[:name].to_s
+            text = truncate(t[:text].inspect, 40)
+
+            if color
+              puts "  #{pos.rjust(8)}  \e[36m#{name.ljust(15)}\e[0m  #{text}"
+            else
+              puts "  #{pos.rjust(8)}  #{name.ljust(15)}  #{text}"
+            end
+          end
+
+          puts
+          puts "#{tokens.size} tokens"
+        end
+
+        def self.truncate(str, max)
+          if str.length > max
+            "#{str[0, max - 3]}..."
+          else
+            str
+          end
+        end
+      end
+
+      # Disassemble a .lkt1 file
+      module Disasm
+        def self.run(argv)
+          options = { dfa: nil, jump: nil, keyword: nil }
+
+          parser = OptionParser.new do |opts|
+            opts.banner = "Usage: lexer_kit disasm [options] <file.lkt1|file.lkb1>"
+
+            opts.on("--dfa INDEX", Integer, "Show DFA table at INDEX") do |v|
+              options[:dfa] = v
+            end
+
+            opts.on("--jump INDEX", Integer, "Show jump table at INDEX") do |v|
+              options[:jump] = v
+            end
+
+            opts.on("--keyword INDEX", Integer, "Show keyword table at INDEX") do |v|
+              options[:keyword] = v
+            end
+
+            opts.on("-h", "--help", "Show this help") do
+              puts opts
+              return 0
+            end
+          end
+
+          parser.parse!(argv)
+
+          if argv.empty?
+            warn "error: No input file specified"
+            warn parser.banner
+            return 1
+          end
+
+          path = argv.shift
+          program = CLI.load_lexer(path)
+
+          require "lexer_kit/debug"
+
+          if options[:dfa]
+            dfa = program.dfa_tables[options[:dfa]]
+            raise ArgumentError, "DFA table #{options[:dfa]} not found" unless dfa
+
+            puts Debug::Visualizer.format_dfa(dfa, program: program)
+          elsif options[:jump]
+            table = program.jump_tables[options[:jump]]
+            raise ArgumentError, "Jump table #{options[:jump]} not found" unless table
+
+            puts Debug::Visualizer.format_jump_table(table)
+          elsif options[:keyword]
+            table = program.keyword_tables[options[:keyword]]
+            raise ArgumentError, "Keyword table #{options[:keyword]} not found" unless table
+
+            puts Debug::Visualizer.format_keyword_table(table, program: program)
+          else
+            puts Debug::Disassembler.new(program).disassemble
+          end
+
+          0
+        end
+      end
+
+      # Visualize DFA as Graphviz DOT
+      module Visualize
+        def self.run(argv)
+          options = { dfa: 0, output: nil }
+
+          parser = OptionParser.new do |opts|
+            opts.banner = "Usage: lexer_kit visualize [options] <file.lkt1|file.lkb1>"
+
+            opts.on("--dfa INDEX", Integer, "DFA table index (default: 0)") do |v|
+              options[:dfa] = v
+            end
+
+            opts.on("-o", "--output FILE", "Output file (default: stdout)") do |v|
+              options[:output] = v
+            end
+
+            opts.on("-h", "--help", "Show this help") do
+              puts opts
+              puts
+              puts "Examples:"
+              puts "  lexer_kit visualize lexer.lkt1 | dot -Tpng -o dfa.png"
+              puts "  lexer_kit visualize --dfa 1 lexer.lkt1 -o dfa.dot"
+              return 0
+            end
+          end
+
+          parser.parse!(argv)
+
+          if argv.empty?
+            warn "error: No input file specified"
+            warn parser.banner
+            return 1
+          end
+
+          path = argv.shift
+          program = CLI.load_lexer(path)
+
+          require "lexer_kit/debug"
+
+          dfa = program.dfa_tables[options[:dfa]]
+          raise ArgumentError, "DFA table #{options[:dfa]} not found" unless dfa
+
+          dot = Debug::Visualizer.dfa_to_dot(dfa, program: program)
+
+          if options[:output]
+            File.write(options[:output], dot)
+            warn "Written: #{options[:output]}"
+          else
+            puts dot
+          end
+
+          0
+        end
+      end
+    end
+  end
+end

data/lib/lexer_kit/cli.rb

@@ -0,0 +1,88 @@
+# frozen_string_literal: true
+
+require "optparse"
+require "json"
+
+module LexerKit
+  # Command-line interface for LexerKit
+  module CLI
+    VERSION = LexerKit::VERSION
+
+    class << self
+      def run(argv)
+        if argv.empty? || argv[0] == "-h" || argv[0] == "--help"
+          print_help
+          return 0
+        end
+
+        if ["-v", "--version"].include?(argv[0])
+          puts "lexer_kit #{VERSION}"
+          return 0
+        end
+
+        command = argv.shift
+        case command
+        when "compile"
+          Commands::Compile.run(argv)
+        when "lex"
+          Commands::Lex.run(argv)
+        when "info"
+          Commands::Info.run(argv)
+        when "disasm"
+          Commands::Disasm.run(argv)
+        when "visualize"
+          Commands::Visualize.run(argv)
+        else
+          warn "Unknown command: #{command}"
+          warn "Run 'lexer_kit --help' for usage."
+          1
+        end
+      rescue StandardError => e
+        render_error(e)
+        1
+      end
+
+      def render_error(error, color: $stderr.tty?)
+        if error.respond_to?(:render)
+          warn error.render(color: color)
+        else
+          warn "error: #{error.message}"
+        end
+      end
+
+      def print_help
+        puts <<~HELP
+          Usage: lexer_kit <command> [options] [arguments]
+
+          Commands:
+            compile   Compile DSL file to .lkt1 or .lkb1
+            lex       Tokenize input with a lexer
+            info      Show lexer information
+            disasm    Disassemble a .lkt1 or .lkb1 file
+            visualize Output DFA as Graphviz DOT format
+
+          Options:
+            -h, --help     Show this help
+            -v, --version  Show version
+
+          Examples:
+            lexer_kit compile json_lexer.rb
+            lexer_kit compile json_lexer.rb --verbose  # show conflict warnings
+            lexer_kit compile json_lexer.rb --dry-run  # check conflicts only
+            lexer_kit lex json_lexer.lkt1 data.json
+            lexer_kit info json_lexer.lkt1
+            lexer_kit lex json_lexer.lkb1 data.json
+            lexer_kit info json_lexer.lkb1
+            lexer_kit visualize json_lexer.lkt1 | dot -Tpng -o dfa.png
+        HELP
+      end
+    end
+
+    # Load lexer from .lkt1 or .lkb1 file
+    def self.load_lexer(path)
+      LexerKit.load_lexer(path)
+    end
+  end
+end
+
+require_relative "cli/commands"

data/lib/lexer_kit/core/diagnostic.rb

@@ -0,0 +1,103 @@
+# frozen_string_literal: true
+
+module LexerKit
+  module Core
+    # Diagnostic holds error information with position and notes.
+    # It can render a human-readable error message with source context.
+    class Diagnostic
+      attr_reader :level, :message, :span, :notes
+
+      LEVEL_LABELS = {
+        error: "error",
+        warning: "warning",
+        note: "note"
+      }.freeze
+
+      LEVEL_COLORS = {
+        error: "\e[1;31m",    # bold red
+        warning: "\e[1;33m",  # bold yellow
+        note: "\e[1;36m"      # bold cyan
+      }.freeze
+
+      RESET = "\e[0m"
+      BOLD = "\e[1m"
+      BLUE = "\e[34m"
+
+      # @param level [Symbol] :error, :warning, or :note
+      # @param message [String] main error message
+      # @param span [Span] location in source
+      # @param notes [Array<String>, nil] additional notes
+      def initialize(level:, message:, span:, notes: nil)
+        @level = level
+        @message = message
+        @span = span
+        @notes = notes&.freeze
+      end
+
+      # Render the diagnostic with source context
+      # @param source [Source]
+      # @param color [Boolean] enable ANSI colors
+      # @return [String]
+      def render(source, color: $stdout.tty?)
+        lines = []
+        line_num, col = source.line_col(@span.start)
+
+        # Header: filename:line:col: level: message
+        loc = source.filename ? "#{source.filename}:" : ""
+        loc += "#{line_num}:#{col}"
+
+        if color
+          level_str = "#{LEVEL_COLORS[@level]}#{LEVEL_LABELS[@level]}#{RESET}"
+          lines << "#{BOLD}#{loc}:#{RESET} #{level_str}: #{BOLD}#{@message}#{RESET}"
+        else
+          lines << "#{loc}: #{LEVEL_LABELS[@level]}: #{@message}"
+        end
+
+        # Source line
+        line_content = source.line_slice(line_num)
+        if line_content
+          line_num_str = line_num.to_s
+          gutter_width = line_num_str.length
+
+          # Line number gutter
+          lines << if color
+                     " #{BLUE}#{line_num_str} |#{RESET} #{line_content}"
+                   else
+                     " #{line_num_str} | #{line_content}"
+                   end
+
+          # Caret line
+          highlight_len = [@span.len, line_content.length - col + 1].min
+          highlight_len = 1 if highlight_len < 1
+          carets = "^" + ("~" * (highlight_len - 1)) # rubocop:disable Style/StringConcatenation
+          padding = " " * (col - 1)
+
+          lines << if color
+                     " #{' ' * gutter_width} #{BLUE}|#{RESET} #{padding}#{LEVEL_COLORS[@level]}#{carets}#{RESET}"
+                   else
+                     " #{' ' * gutter_width} | #{padding}#{carets}"
+                   end
+        end
+
+        # Notes
+        @notes&.each do |note|
+          lines << if color
+                     "       #{BOLD}note:#{RESET} #{note}"
+                   else
+                     "       note: #{note}"
+                   end
+        end
+
+        lines.join("\n")
+      end
+
+      def to_s
+        "#{LEVEL_LABELS[@level]}: #{@message} at #{@span}"
+      end
+
+      def inspect
+        "#<LexerKit::Core::Diagnostic #{@level} #{@span}: #{@message.inspect}>"
+      end
+    end
+  end
+end