RubyGems - descent - Versions diffs - 0.7.1 - Mend

descent 0.7.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Files changed (22) hide show

checksums.yaml +7 -0
data/CHANGELOG.md +285 -0
data/README.md +583 -0
data/SYNTAX.md +334 -0
data/exe/descent +15 -0
data/lib/descent/ast.rb +69 -0
data/lib/descent/generator.rb +489 -0
data/lib/descent/ir.rb +98 -0
data/lib/descent/ir_builder.rb +1479 -0
data/lib/descent/lexer.rb +308 -0
data/lib/descent/parser.rb +450 -0
data/lib/descent/railroad.rb +272 -0
data/lib/descent/templates/rust/_command.liquid +174 -0
data/lib/descent/templates/rust/parser.liquid +1163 -0
data/lib/descent/tools/debug.rb +115 -0
data/lib/descent/tools/diagram.rb +48 -0
data/lib/descent/tools/generate.rb +47 -0
data/lib/descent/tools/validate.rb +56 -0
data/lib/descent/validator.rb +231 -0
data/lib/descent/version.rb +5 -0
data/lib/descent.rb +34 -0
metadata +101 -0

data/lib/descent/railroad.rb ADDED Viewed

@@ -0,0 +1,272 @@
+# frozen_string_literal: true
+module Descent
+  # Generates Python code using the railroad-diagrams library.
+  #
+  # This is a SPIKE - quick proof-of-concept, not production quality.
+  # The railroad-diagrams library is what JSON.org uses for its diagrams.
+  #
+  # Usage:
+  #   pip install railroad-diagrams
+  #   descent diagram parser.desc > diagram.py
+  #   python diagram.py > diagram.svg
+  #
+  # Mapping from .desc concepts to railroad primitives:
+  #   - Function → one Diagram per function
+  #   - State cases → Choice (alternatives)
+  #   - Self-looping state → ZeroOrMore or OneOrMore
+  #   - c[...] match → Terminal (circles)
+  #   - /function call → NonTerminal (rectangles)
+  #   - |return → end of path
+  #   - |>> → loop/continue
+  class Railroad
+    def initialize(ir) = @ir = ir
+    def generate
+      lines = []
+      lines << header
+      lines << ''
+      @ir.functions.each do |func|
+        lines << function_diagram(func)
+        lines << ''
+      end
+      lines << footer
+      lines.join("\n")
+    end
+    private
+    def header
+      <<~PY
+        #!/usr/bin/env python3
+        """
+        Railroad diagrams for #{@ir.name} parser.
+        Generated by descent - run this to produce SVG.
+        Usage:
+          python #{@ir.name}_diagram.py > #{@ir.name}_diagram.svg
+        Or for individual function SVGs:
+          python #{@ir.name}_diagram.py --split
+        Requires: pip install railroad-diagrams
+        """
+        from railroad import Diagram, Choice, Sequence, Optional, ZeroOrMore, OneOrMore
+        from railroad import Terminal, NonTerminal, Comment, Skip, Start, End
+        import sys
+        DIAGRAMS = {}
+      PY
+    end
+    def footer
+      <<~PY
+        def main():
+            import railroad
+            if '--split' in sys.argv:
+                # Output each function to separate file
+                for name, diag in DIAGRAMS.items():
+                    with open(f'{name}.svg', 'w') as f:
+                        diag.writeStandalone(f.write)
+                    print(f'Wrote {name}.svg', file=sys.stderr)
+            else:
+                # Output all diagrams as single HTML with proper CSS
+                print('<!DOCTYPE html><html><head>')
+                print(f'<style>{railroad.DEFAULT_STYLE}</style>')
+                print('<style>svg { margin: 20px; display: block; } h2 { font-family: sans-serif; }</style>')
+                print('</head><body>')
+                for name, diag in DIAGRAMS.items():
+                    print(f'<h2>{name}</h2>')
+                    diag.writeSvg(print)
+                print('</body></html>')
+        if __name__ == '__main__':
+            main()
+      PY
+    end
+    def function_diagram(func)
+      # Build the diagram for this function
+      # Analyze state transitions to build proper railroad structure
+      states = func.states
+      return "# #{func.name}: no states" if states.empty?
+      # Build a map of state names to states for lookup
+      state_map = states.to_h { |s| [s.name, s] }
+      # Start from the first state and build the diagram
+      diagram_content = build_state_sequence(states.first, state_map, Set.new)
+      # Add return type info as comment if present
+      comment = func.return_type ? ", Comment('→ #{func.return_type}')" : ''
+      <<~PY
+        # Function: #{func.name}#{"(#{func.params.join(', ')})" unless func.params.empty?}
+        DIAGRAMS['#{func.name}'] = Diagram(
+            #{diagram_content}#{comment}
+        )
+      PY
+    end
+    def build_state_sequence(state, state_map, visited)
+      return 'Skip()' if state.nil? || visited.include?(state.name)
+      visited += [state.name]
+      cases   = state.cases.reject(&:conditional?)
+      return 'Skip()' if cases.empty?
+      # Categorize cases by their transition type
+      exit_cases    = []      # Cases that return (exit the function)
+      loop_cases    = []      # Cases that self-loop (stay in this state)
+      forward_cases = [] # Cases that go to another state
+      cases.each do |kase|
+        transition = find_transition(kase)
+        # Normalize transition: remove leading colon if present
+        transition = transition.sub(/^:/, '') if transition
+        if returns?(kase)
+          exit_cases << kase
+        elsif transition.nil? || transition.empty? || transition == state.name
+          loop_cases << kase
+        else
+          forward_cases << [kase, transition]
+        end
+      end
+      parts = []
+      # If we have forward transitions, they form a sequence
+      # Group cases by their target state
+      if forward_cases.any?
+        # Cases that go to the same next state can be shown as Choice
+        by_target = forward_cases.group_by { |_, target| target }
+        by_target.each do |target_state, cases_to_target|
+          entry_cases = cases_to_target.map(&:first)
+          if entry_cases.size == 1
+            parts << case_to_element(entry_cases.first, nil)
+          else
+            elements = entry_cases.map { |c| case_to_element(c, nil) }
+            parts << "Choice(0, #{elements.join(', ')})"
+          end
+          # Recurse into the target state
+          if state_map[target_state] && !visited.include?(target_state)
+            next_part = build_state_sequence(state_map[target_state], state_map, visited)
+            parts << next_part unless next_part == 'Skip()'
+          end
+        end
+      end
+      # Loop cases become ZeroOrMore
+      if loop_cases.any?
+        if loop_cases.size == 1
+          loop_content = case_to_element(loop_cases.first, nil)
+        else
+          elements     = loop_cases.map { |c| case_to_element(c, nil) }
+          loop_content = "Choice(0, #{elements.join(', ')})"
+        end
+        parts << "ZeroOrMore(#{loop_content})"
+      end
+      # Exit cases become the terminator
+      if exit_cases.any?
+        if exit_cases.size == 1
+          parts << case_to_element(exit_cases.first, nil)
+        else
+          elements = exit_cases.map { |c| case_to_element(c, nil) }
+          parts << "Choice(0, #{elements.join(', ')})"
+        end
+      end
+      return 'Skip()' if parts.empty?
+      return parts.first if parts.size == 1
+      "Sequence(#{parts.join(', ')})"
+    end
+    def find_transition(kase)
+      kase.commands.each do |cmd|
+        return cmd.args[:value] || cmd.args['value'] if cmd.type == :transition
+      end
+      nil
+    end
+    def returns?(kase) = kase.commands.any? { |cmd| cmd.type == :return }
+    # Known character class patterns - map expanded chars back to names
+    CHAR_CLASS_NAMES = {
+      'abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ' => 'LETTER',
+      'abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789_-' => 'LABEL_CONT',
+      '0123456789' => 'DIGIT',
+      '0123456789abcdefABCDEF' => 'HEX_DIGIT',
+      " \t" => 'WS',
+      'abcdefghijklmnopqrstuvwxyz' => 'a-z',
+      'ABCDEFGHIJKLMNOPQRSTUVWXYZ' => 'A-Z'
+    }.freeze
+    def case_to_element(kase, _func)
+      # Build element for a single case
+      parts = []
+      # What we're matching
+      if kase.default?
+        parts << 'Skip()' # Default case = any other char
+      elsif kase.chars && !kase.chars.empty?
+        # Character match - show as terminal
+        chars_display = format_chars(kase.chars)
+        parts << "Terminal(#{chars_display.inspect})"
+      elsif kase.special_class
+        # Special class like LETTER, DIGIT
+        parts << "NonTerminal(#{kase.special_class.to_s.upcase.inspect})"
+      elsif kase.param_ref
+        # Parameter reference
+        parts << "NonTerminal(':#{kase.param_ref}')"
+      end
+      # What actions we take - look for function calls
+      kase.commands.each do |cmd|
+        case cmd.type
+        when :call
+          func_name = cmd.args[:name]
+          parts << "NonTerminal(#{func_name.inspect})"
+        when :emit
+          # Inline emit - show as comment
+          event_type = cmd.args[:type]
+          parts << "Comment(#{event_type.inspect})" if event_type
+        when :keywords
+          # Keywords lookup
+          kw_name = cmd.args[:name]
+          parts << "NonTerminal('KEYWORDS(#{kw_name})')"
+        end
+      end
+      return parts.first if parts.size == 1
+      "Sequence(#{parts.join(', ')})"
+    end
+    def format_chars(chars)
+      if chars.size == 1
+        escape_char(chars.first)
+      else
+        # Check if this matches a known class
+        joined = chars.join
+        CHAR_CLASS_NAMES[joined] || chars.map { |c| escape_char(c) }.join('')
+      end
+    end
+    def escape_char(char)
+      case char
+      when "\n" then '\\n'
+      when "\t" then '\\t'
+      when "\r" then '\\r'
+      when ' '  then '␣'
+      else char
+      end
+    end
+  end
+end

data/lib/descent/templates/rust/_command.liquid ADDED Viewed

@@ -0,0 +1,174 @@
+{% comment %}
+  Command partial - generates Rust code for a single IR::Command
+  Variables in scope:
+  - cmd: the command hash {type, args}
+  - func: the function hash
+  - return_type_info: the function's return type info (or nil)
+  - states: array of states (for multi-state functions)
+{% endcomment %}
+{% case cmd.type %}
+{% when "advance" %}
+                    self.advance();
+{% when "advance_to" %}
+{% comment %} Explicit advance-to: ->[chars] uses memchr to scan forward.
+              Limited to 1-6 chars (validated by IR builder). {% endcomment %}
+{% assign target_chars = cmd.args.value | split: "" %}
+{% assign char_count = target_chars | size %}
+{% if char_count == 1 %}
+                    self.scan_to1({{ target_chars[0] | escape_rust_char }});
+{% elsif char_count == 2 %}
+                    self.scan_to2({{ target_chars[0] | escape_rust_char }}, {{ target_chars[1] | escape_rust_char }});
+{% elsif char_count == 3 %}
+                    self.scan_to3({{ target_chars[0] | escape_rust_char }}, {{ target_chars[1] | escape_rust_char }}, {{ target_chars[2] | escape_rust_char }});
+{% elsif char_count == 4 %}
+                    self.scan_to4({{ target_chars[0] | escape_rust_char }}, {{ target_chars[1] | escape_rust_char }}, {{ target_chars[2] | escape_rust_char }}, {{ target_chars[3] | escape_rust_char }});
+{% elsif char_count == 5 %}
+                    self.scan_to5({{ target_chars[0] | escape_rust_char }}, {{ target_chars[1] | escape_rust_char }}, {{ target_chars[2] | escape_rust_char }}, {{ target_chars[3] | escape_rust_char }}, {{ target_chars[4] | escape_rust_char }});
+{% elsif char_count == 6 %}
+                    self.scan_to6({{ target_chars[0] | escape_rust_char }}, {{ target_chars[1] | escape_rust_char }}, {{ target_chars[2] | escape_rust_char }}, {{ target_chars[3] | escape_rust_char }}, {{ target_chars[4] | escape_rust_char }}, {{ target_chars[5] | escape_rust_char }});
+{% else %}
+                    unreachable!("advance_to validated to 1-6 chars");
+{% endif %}
+{% when "mark" %}
+                    self.mark();
+{% when "term" %}
+{% assign term_offset = cmd.args.offset | default: 0 %}
+                    self.set_term({{ term_offset }});
+{% when "prepend" %}
+{% comment %} Prepend literal bytes to the accumulation buffer. Empty literals are
+              filtered as :noop at parse time, so we always have content here. {% endcomment %}
+                    self.prepend_bytes(b"{{ cmd.args.literal }}");
+{% when "prepend_param" %}
+{% comment %} Prepend parameter bytes to accumulation buffer. Parameter is &'static [u8],
+              so empty slice is naturally a no-op, NUL bytes work fine. {% endcomment %}
+{% assign param_name = cmd.args.param_ref %}
+                    self.prepend_bytes({{ param_name }});
+{% when "return" %}
+{% comment %} Emit appropriate event based on return type, then return {% endcomment %}
+{% if cmd.args.return_value %}
+{% comment %} INTERNAL type returning a computed value {% endcomment %}
+                    return {{ cmd.args.return_value }};
+{% elsif cmd.args.emit_type %}
+{% comment %} Explicit return type specified {% endcomment %}
+{% if cmd.args.emit_mode == "mark" %}
+                    on_event(Event::{{ cmd.args.emit_type }} { content: self.term(), span: self.span_from_mark() });
+{% elsif cmd.args.emit_mode == "literal" %}
+                    on_event(Event::{{ cmd.args.emit_type }} { content: std::borrow::Cow::Borrowed(b"{{ cmd.args.literal }}"), span: self.span() });
+{% else %}
+                    on_event(Event::{{ cmd.args.emit_type }} { content: std::borrow::Cow::Borrowed(b""), span: self.span() });
+{% endif %}
+                    return;
+{% elsif return_type_info.kind == "internal" %}
+{% comment %} INTERNAL type with no explicit return value - return 0 as default {% endcomment %}
+                    return 0;
+{% elsif cmd.args.suppress_auto_emit and return_type_info.kind == "content" %}
+{% comment %} Fix #11: Inline emit already happened for CONTENT type - skip auto-emit.
+              Note: BRACKET types ALWAYS emit End event regardless of inline emits,
+              because End is structural, not a content event. {% endcomment %}
+                    return;
+{% elsif return_type_info.kind == "content" %}
+                    on_event(Event::{{ func.return_type }} { content: self.term(), span: self.span_from_mark() });
+                    return;
+{% elsif return_type_info.kind == "bracket" %}
+                    on_event(Event::{{ func.return_type }}End { span: self.span() });
+                    return;
+{% else %}
+                    return;
+{% endif %}
+{% when "transition" %}
+{% if cmd.args.value == "" or cmd.args.value == nil %}
+                    {% comment %} Self-loop - continue {% endcomment %}
+                    continue;
+{% else %}
+                    state = State::{{ cmd.args.value | remove: ":" | pascalcase }};
+                    continue;
+{% endif %}
+{% when "call" %}
+{% comment %} Function call - use pre-parsed name and call_args from IR {% endcomment %}
+{% if cmd.args.is_error %}
+{% comment %} Built-in /error(ErrorCode) - emit error event (caller adds |return if needed) {% endcomment %}
+{% if cmd.args.call_args %}
+                    on_event(Event::Error { code: ParseErrorCode::{{ cmd.args.call_args | pascalcase }}, span: self.span() });
+{% else %}
+                    on_event(Event::Error { code: ParseErrorCode::UnexpectedChar, span: self.span() });
+{% endif %}
+{% elsif cmd.args.call_args %}
+                    self.parse_{{ cmd.args.name }}({{ cmd.args.call_args | rust_expr }}, on_event);
+{% else %}
+                    self.parse_{{ cmd.args.name }}(on_event);
+{% endif %}
+{% when "assign" %}
+{% if cmd.args.var and cmd.args.expr %}
+                    {{ cmd.args.var }} = {{ cmd.args.expr | rust_expr }};
+{% endif %}
+{% when "add_assign" %}
+{% if cmd.args.var and cmd.args.expr %}
+                    {{ cmd.args.var }} += {{ cmd.args.expr | rust_expr }};
+{% endif %}
+{% when "sub_assign" %}
+{% if cmd.args.var and cmd.args.expr %}
+                    {{ cmd.args.var }} -= {{ cmd.args.expr | rust_expr }};
+{% endif %}
+{% when "emit" %}
+{% comment %} Explicit emit - used for mid-function emissions {% endcomment %}
+{% assign emit_type = cmd.args.value %}
+                    on_event(Event::{{ emit_type }} { content: self.term(), span: self.span_from_mark() });
+{% when "inline_emit_bare" %}
+{% comment %} Inline emit with no payload: TypeName - CONTENT types still need content field {% endcomment %}
+                    on_event(Event::{{ cmd.args.type }} { content: std::borrow::Cow::Borrowed(b""), span: self.span() });
+{% when "inline_emit_mark" %}
+{% comment %} Inline emit using accumulated content: TypeName(USE_MARK) {% endcomment %}
+                    on_event(Event::{{ cmd.args.type }} { content: self.term(), span: self.span_from_mark() });
+{% when "inline_emit_literal" %}
+{% comment %} Inline emit with literal: TypeName(literal) - strip $ sigil if present {% endcomment %}
+{% assign literal = cmd.args.literal | remove_first: '$' %}
+                    on_event(Event::{{ cmd.args.type }} { content: std::borrow::Cow::Borrowed(b"{{ literal }}"), span: self.span() });
+{% when "error" %}
+{% assign error_code = cmd.args.value | pascalcase %}
+                    on_event(Event::Error { code: ParseErrorCode::{{ error_code }}, span: self.span() });
+{% when "keywords_lookup" %}
+{% comment %} Look up accumulated content in keyword map, call fallback if not found {% endcomment %}
+                    self.lookup_{{ cmd.args.name }}_or_fallback(on_event);
+{% when "conditional" %}
+{% comment %} Inline conditional: if/else chain {% endcomment %}
+{% for clause in cmd.args.clauses %}
+{% if forloop.first %}
+                    if {{ clause.condition | rust_expr }} {
+{% else %}
+{% if clause.condition %}
+                    } else if {{ clause.condition | rust_expr }} {
+{% else %}
+                    } else {
+{% endif %}
+{% endif %}
+{% for nested_cmd in clause.commands %}
+{% include 'command' cmd: nested_cmd, func: func, return_type_info: return_type_info, states: states %}
+{% endfor %}
+{% endfor %}
+                    }
+{% else %}
+                    compile_error!("Unknown command type passed to template: {{ cmd.type }}");
+{% endcase %}