descent 0.7.1

data/lib/descent/tools/debug.rb

@@ -0,0 +1,115 @@
+# frozen_string_literal: true
+
+desc 'Debug/inspect .desc file parsing stages'
+long_desc <<~DESC
+  Shows the intermediate parsing stages for a .desc file:
+  tokens, AST, and IR. Useful for debugging grammar issues.
+
+  Examples:
+    descent debug parser.desc           # Show all stages
+    descent debug parser.desc --tokens  # Show tokens only
+    descent debug parser.desc --ast     # Show AST only
+    descent debug parser.desc --ir      # Show IR only
+DESC
+
+required_arg :file, desc: '.desc specification file'
+flag :tokens, '--tokens', desc: 'Show tokens only'
+flag :ast, '--ast', desc: 'Show AST only'
+flag :ir, '--ir', desc: 'Show IR only'
+
+def run
+  require 'descent'
+
+  unless File.exist?(file)
+    Output.error "File not found: #{file}"
+    exit 1
+  end
+
+  # Default: show all if no specific flag
+  show_all = !tokens && !ast && !ir
+
+  begin
+    content    = File.read(file)
+    token_list = Descent::Lexer.new(content, source_file: file).tokenize
+
+    if tokens || show_all
+      Output.header 'TOKENS'
+      token_list.each_with_index do |t, i|
+        puts format('%3d: L%-3d %-12s id=%-20s rest=%s',
+                    i, t.lineno, t.tag.inspect, t.id.inspect, t.rest.inspect)
+      end
+      puts
+    end
+
+    ast_result = Descent::Parser.new(token_list).parse
+
+    if ast || show_all
+      Output.header 'AST'
+      puts "Parser: #{ast_result.name.inspect}"
+      puts "Entry:  #{ast_result.entry_point.inspect}"
+      puts
+      puts 'Types:'
+      ast_result.types.each { |t| puts "  #{t.name}: #{t.kind}" }
+      puts
+      puts 'Functions:'
+      ast_result.functions.each do |f|
+        puts "  #{f.name}#{":#{f.return_type}" if f.return_type} #{f.params.map { |p| ":#{p}" }.join(' ')}"
+        f.states.each do |s|
+          puts "    state[:#{s.name}]"
+          s.cases.each do |c|
+            chars_desc = c.chars.nil? ? 'default' : "c[#{c.chars}]"
+            puts "      #{chars_desc}#{" .#{c.substate}" if c.substate}"
+            c.commands.each { |cmd| puts "        #{cmd.type}: #{cmd.value.inspect}" }
+          end
+        end
+      end
+      puts
+    end
+
+    if ir || show_all
+      ir_result = Descent::IRBuilder.new(ast_result).build
+
+      Output.header 'IR'
+      puts "Parser: #{ir_result.name.inspect}"
+      puts "Entry:  #{ir_result.entry_point.inspect}"
+      puts
+      puts 'Types:'
+      ir_result.types.each do |t|
+        flags = []
+        flags << 'emits_start' if t.emits_start
+        flags << 'emits_end' if t.emits_end
+        puts "  #{t.name}: #{t.kind} #{flags.join(', ')}"
+      end
+      puts
+      puts 'Functions:'
+      ir_result.functions.each do |f|
+        puts "  #{f.name}#{":#{f.return_type}" if f.return_type} #{f.params.map { |p| ":#{p}" }.join(' ')}"
+        puts "    emits_events: #{f.emits_events}"
+        puts "    expects_char: #{f.expects_char.inspect}" if f.expects_char
+        puts "    locals: #{f.locals.inspect}" unless f.locals.empty?
+        f.states.each do |s|
+          scan_info = s.scannable? ? " SCAN(#{s.scan_chars.map(&:inspect).join(', ')})" : ''
+          puts "    state[:#{s.name}]#{scan_info}"
+          s.cases.each do |c|
+            chars_desc = if c.conditional?
+                           "if[#{c.condition}]"
+                         elsif c.default?
+                           'default'
+                         elsif c.special_class
+                           c.special_class.to_s.upcase
+                         elsif c.chars
+                           "c[#{c.chars.map { |ch| ch == "\n" ? '\\n' : ch }.join}]"
+                         else
+                           'c[???]'
+                         end
+            puts "      #{chars_desc}#{" .#{c.substate}" if c.substate}"
+            c.commands.each { |cmd| puts "        #{cmd.type}: #{cmd.args.inspect}" }
+          end
+        end
+      end
+    end
+  rescue Descent::Error => e
+    Output.error e.message
+    exit 1
+  end
+end

data/lib/descent/tools/diagram.rb

@@ -0,0 +1,48 @@
+# frozen_string_literal: true
+
+desc 'Generate railroad diagram from .desc file'
+long_desc <<~DESC
+  Generates Python code that produces railroad diagrams (like JSON.org).
+  The output uses the railroad-diagrams library.
+
+  Examples:
+    descent diagram parser.desc > diagram.py
+    python diagram.py > diagram.html       # All diagrams as HTML
+    python diagram.py --split              # Each function as separate SVG
+
+  Requires: pip install railroad-diagrams
+DESC
+
+required_arg :file, desc: '.desc specification file'
+flag :output, '-o FILE', '--output=FILE', desc: 'Output file (default: stdout)'
+
+def run
+  require 'descent'
+  require 'descent/railroad'
+
+  unless File.exist?(file)
+    Output.error "File not found: #{file}"
+    exit 1
+  end
+
+  begin
+    content     = File.read(file)
+    source_file = file
+
+    tokens = Descent::Lexer.new(content, source_file:).tokenize
+    ast    = Descent::Parser.new(tokens).parse
+    ir     = Descent::IRBuilder.new(ast).build
+
+    code = Descent::Railroad.new(ir).generate
+
+    if output
+      File.write(output, code)
+      Output.success "Generated #{output}"
+    else
+      puts code
+    end
+  rescue Descent::Error => e
+    Output.error e.message
+    exit 1
+  end
+end

data/lib/descent/tools/generate.rb

@@ -0,0 +1,47 @@
+# frozen_string_literal: true
+
+desc 'Generate parser from .desc file'
+long_desc <<~DESC
+  Parses the .desc specification file and generates a parser in the
+  target language. Currently supports Rust output.
+
+  Examples:
+    descent generate parser.desc
+    descent generate parser.desc -o src/parser.rs
+    descent generate parser.desc --target rust
+    descent generate parser.desc --trace
+DESC
+
+required_arg :file, desc: '.desc specification file'
+flag :output, '-o FILE', '--output=FILE', desc: 'Output file (default: stdout)'
+flag :target, '-t TARGET', '--target=TARGET', desc: 'Target language: rust, c (default: rust)'
+flag :trace, '--trace', desc: 'Enable trace output in generated parser'
+
+def run
+  require 'descent'
+
+  unless File.exist?(file)
+    Output.error "File not found: #{file}"
+    exit 1
+  end
+
+  target_sym = (target || 'rust').to_sym
+  trace_opt  = trace ? true : false
+
+  begin
+    code = Descent.generate(file, target: target_sym, trace: trace_opt)
+
+    # Clean up excessive whitespace
+    code = code.gsub(/\n{3,}/, "\n\n")
+
+    if output
+      File.write(output, code)
+      Output.success "Generated #{output}"
+    else
+      puts code
+    end
+  rescue Descent::Error => e
+    Output.error e.message
+    exit 1
+  end
+end

data/lib/descent/tools/validate.rb

@@ -0,0 +1,56 @@
+# frozen_string_literal: true
+
+desc 'Validate .desc file without generating'
+long_desc <<~DESC
+  Parses and validates the .desc specification file, checking for:
+  - Syntax errors
+  - Undefined type references
+  - Undefined function references
+  - Empty states
+  - Other semantic issues
+
+  Examples:
+    descent validate parser.desc
+DESC
+
+required_arg :file, desc: '.desc specification file'
+
+def run
+  require 'descent'
+
+  unless File.exist?(file)
+    Output.error "File not found: #{file}"
+    exit 1
+  end
+
+  begin
+    content = File.read(file)
+    tokens  = Descent::Lexer.new(content, source_file: file).tokenize
+    ast     = Descent::Parser.new(tokens).parse
+    ir      = Descent::IRBuilder.new(ast).build
+
+    result  = Descent::Validator.new(ir).validate
+
+    if result.errors.any?
+      Output.error "Validation failed with #{result.errors.size} error(s):"
+      result.errors.each { |e| Output.bullet e }
+      exit 1
+    end
+
+    if result.warnings.any?
+      Output.warn "Validation passed with #{result.warnings.size} warning(s):"
+      result.warnings.each { |w| Output.bullet w }
+    else
+      Output.success "Validation passed: #{file}"
+    end
+
+    # Show summary
+    puts
+    puts "Types:     #{ir.types.size}"
+    puts "Functions: #{ir.functions.size}"
+    puts "States:    #{ir.functions.sum { |f| f.states.size }}"
+  rescue Descent::Error => e
+    Output.error e.message
+    exit 1
+  end
+end

data/lib/descent/validator.rb

@@ -0,0 +1,231 @@
+# frozen_string_literal: true
+
+module Descent
+  # Validates IR and collects warnings/errors.
+  #
+  # Run after IR building to catch issues before code generation.
+  class Validator
+    Warning = Data.define(:message, :location)
+    Error   = Data.define(:message, :location)
+
+    attr_reader :warnings, :errors
+
+    def initialize(ir)
+      @ir       = ir
+      @warnings = []
+      @errors   = []
+    end
+
+    def validate
+      validate_parser_name
+      validate_types
+      validate_functions
+      validate_entry_point
+
+      self
+    end
+
+    def valid? = @errors.empty?
+
+    def report
+      output = []
+
+      @errors.each do |e|
+        loc = e.location ? " (#{e.location})" : ''
+        output << "ERROR#{loc}: #{e.message}"
+      end
+
+      @warnings.each do |w|
+        loc = w.location ? " (#{w.location})" : ''
+        output << "WARNING#{loc}: #{w.message}"
+      end
+
+      output.join("\n")
+    end
+
+    private
+
+    def warn(message, location: nil) = @warnings << Warning.new(message:, location:)
+
+    def error(message, location: nil) = @errors << Error.new(message:, location:)
+
+    # ========== Parser Name Validation ==========
+
+    def validate_parser_name
+      return if @ir.name && !@ir.name.empty?
+
+      error 'Missing parser name: add |parser <name> directive',
+            location: 'L1'
+    end
+
+    # ========== Type Validation ==========
+
+    def validate_types
+      seen_types = {}
+
+      @ir.types.each do |type|
+        loc = "L#{type.lineno}"
+
+        # Check for duplicate type names
+        if seen_types[type.name]
+          error "Duplicate type declaration: #{type.name}",
+                location: loc
+        end
+        seen_types[type.name] = type
+
+        # Check for valid kind
+        unless %i[bracket content internal].include?(type.kind)
+          error "Unknown type kind '#{type.kind}' for #{type.name}",
+                location: loc
+        end
+      end
+    end
+
+    # ========== Function Validation ==========
+
+    def validate_functions
+      seen_functions = {}
+
+      @ir.functions.each do |func|
+        loc = "L#{func.lineno}"
+
+        # Check for duplicate function names
+        if seen_functions[func.name]
+          warn "Duplicate function definition: #{func.name}",
+               location: loc
+        end
+        seen_functions[func.name] = func
+
+        # Check for empty functions (no states)
+        if func.states.empty?
+          warn "Function '#{func.name}' has no states",
+               location: loc
+        end
+
+        # Check return type exists
+        if func.return_type && !type_exists?(func.return_type)
+          warn "Return type '#{func.return_type}' not declared",
+               location: loc
+        end
+
+        validate_function_states(func)
+      end
+    end
+
+    def validate_function_states(func)
+      func.states.each do |state|
+        loc = "L#{state.lineno}"
+
+        # Check for empty states (no cases)
+        if state.cases.empty? && state.eof_handler.nil?
+          warn "State '#{state.name}' in #{func.name} has no cases",
+               location: loc
+        end
+
+        # NOTE: EOF handling is inferred per spec, so we don't warn about missing handlers
+        # The code generator will infer appropriate EOF behavior based on:
+        # - MARK status (emit content if marked)
+        # - Return type (BRACKET types emit End, CONTENT types emit content)
+        # - EXPECTS annotation (error if not satisfied)
+
+        validate_state_cases(func, state)
+      end
+    end
+
+    def validate_state_cases(func, state)
+      loc = "L#{state.lineno}"
+
+      state.cases.each do |kase|
+        # Check for empty character matches
+        if kase.chars&.empty? && !kase.default? && kase.special_class.nil?
+          warn "Empty character match in #{func.name}:#{state.name}",
+               location: loc
+        end
+
+        # Check for cases with no commands (this is actually often valid - just advance)
+        # if kase.commands.empty?
+        #   warn "Case with no commands in #{func.name}:#{state.name}",
+        #        location: loc
+        # end
+
+        validate_commands(func, state, kase.commands, loc)
+      end
+    end
+
+    def validate_commands(func, _state, commands, loc)
+      commands.each do |cmd|
+        case cmd.type
+        when :call
+          # Check if called function exists
+          # IR builder stores name in :name field (not :value)
+          func_name = cmd.args[:name] || cmd.args['name']
+          unless func_name && function_exists?(func_name)
+            warn "Call to undefined function '#{func_name}'",
+                 location: loc
+          end
+        when :emit
+          # Check if emitted type exists
+          emitted = cmd.args[:value] || cmd.args['value']
+          # Strip any suffix like "Start" or "End"
+          base_type = emitted&.sub(/(Start|End|Anon)$/, '')
+          unless type_exists?(base_type) || builtin_emit?(emitted)
+            warn "Emit of undefined type '#{emitted}'",
+                 location: loc
+          end
+        when :transition
+          # Check if target state exists (if specified)
+          target = cmd.args[:value] || cmd.args['value']
+          next if target.nil? || target.empty? # Self-loop, valid
+
+          if target.start_with?(':')
+            # Target is :statename - validate state exists
+            state_name = target.delete_prefix(':')
+            unless state_exists_in_function?(func, state_name)
+              warn "Transition to undefined state '#{target}'",
+                   location: loc
+            end
+          else
+            # Target doesn't start with : but isn't empty - probably malformed
+            warn "Invalid transition target '#{target}' (should be :statename or empty)",
+                 location: loc
+          end
+        end
+      end
+    end
+
+    # ========== Entry Point Validation ==========
+
+    def validate_entry_point
+      return unless @ir.entry_point
+
+      # Parse entry point: /function or /function:state
+      entry = @ir.entry_point.delete_prefix('/')
+      func_name, _state_name = entry.split(':')
+
+      return if function_exists?(func_name)
+
+      error "Entry point references undefined function '#{func_name}'",
+            location: 'entry-point'
+    end
+
+    # ========== Helpers ==========
+
+    def type_exists?(name)
+      return false if name.nil?
+
+      @ir.types.any? { |t| t.name.downcase == name.downcase }
+    end
+
+    def function_exists?(name)
+      return false if name.nil?
+
+      @ir.functions.any? { |f| f.name == name }
+    end
+
+    def state_exists_in_function?(func, state_name) = func.states.any? { |s| s.name == state_name }
+
+    def has_eof_in_cases?(_state) = false
+
+    def builtin_emit?(name) = %w[Error Warning].include?(name)
+  end
+end

data/lib/descent/version.rb

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+module Descent
+  VERSION = '0.7.1'
+end

data/lib/descent.rb

@@ -0,0 +1,34 @@
+# frozen_string_literal: true
+
+require_relative 'descent/version'
+
+module Descent
+  class Error < StandardError; end
+  class ParseError < Error; end
+  class ValidationError < Error; end
+
+  autoload :AST,       'descent/ast'
+  autoload :IR,        'descent/ir'
+  autoload :Lexer,     'descent/lexer'
+  autoload :Parser,    'descent/parser'
+  autoload :IRBuilder, 'descent/ir_builder'
+  autoload :Validator, 'descent/validator'
+  autoload :Generator, 'descent/generator'
+
+  # Main entry point: parse a .desc file and generate output
+  #
+  # @param input [String] Path to .desc file or string content
+  # @param target [Symbol] Target language (:rust, :c)
+  # @param options [Hash] Additional options
+  # @return [String] Generated parser code
+  def self.generate(input, target:, **)
+    content     = File.exist?(input) ? File.read(input) : input
+    source_file = File.exist?(input) ? input : '(string)'
+
+    tokens = Lexer.new(content, source_file:).tokenize
+    ast    = Parser.new(tokens).parse
+    ir     = IRBuilder.new(ast).build
+
+    Generator.new(ir, target:, **).generate
+  end
+end

metadata

@@ -0,0 +1,101 @@
+--- !ruby/object:Gem::Specification
+name: descent
+version: !ruby/object:Gem::Version
+  version: 0.7.1
+platform: ruby
+authors:
+- Joseph Wecker
+autorequire:
+bindir: exe
+cert_chain: []
+date: 2026-01-10 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: devex
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.3'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.3'
+- !ruby/object:Gem::Dependency
+  name: liquid
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '5.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '5.0'
+description: |
+  Generates high-performance callback-based recursive descent parsers from
+  declarative .desc specifications. Supports target language of Rust and will
+  soon support multiple more target languages (Rust, C, wasm, ruby, go,
+  elixir...) via Liquid templates. The .desc format is valid UDON, enabling
+  future bootstrapping where descent can parse its own input format.
+email:
+- joseph.wecker@gmail.com
+executables:
+- descent
+extensions: []
+extra_rdoc_files: []
+files:
+- CHANGELOG.md
+- README.md
+- SYNTAX.md
+- exe/descent
+- lib/descent.rb
+- lib/descent/ast.rb
+- lib/descent/generator.rb
+- lib/descent/ir.rb
+- lib/descent/ir_builder.rb
+- lib/descent/lexer.rb
+- lib/descent/parser.rb
+- lib/descent/railroad.rb
+- lib/descent/templates/rust/_command.liquid
+- lib/descent/templates/rust/parser.liquid
+- lib/descent/tools/debug.rb
+- lib/descent/tools/diagram.rb
+- lib/descent/tools/generate.rb
+- lib/descent/tools/validate.rb
+- lib/descent/validator.rb
+- lib/descent/version.rb
+homepage: https://github.com/v2-io/descent
+licenses:
+- MIT
+metadata:
+  homepage_uri: https://github.com/v2-io/descent
+  source_code_uri: https://github.com/v2-io/descent
+  changelog_uri: https://github.com/v2-io/descent/blob/main/CHANGELOG.md
+  rubygems_mfa_required: 'true'
+post_install_message:
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: 3.3.0
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 3.5.22
+signing_key:
+specification_version: 4
+summary: Recursive descent parser generator from .desc specifications
+test_files: []