ibex 0.1.0 → 0.2.0

data/lib/ibex/ir/grammar_ir.rb

@@ -2,7 +2,8 @@
 
 module Ibex
   module IR
-    SCHEMA_VERSION = 1
+    SCHEMA_VERSION = 2
+    SUPPORTED_SCHEMA_VERSIONS = [1, 2].freeze #: Array[Integer]
 
     # @rbs (untyped value) -> untyped
     def deep_freeze(value)
@@ -26,16 +27,23 @@ module Ibex
       attr_reader :reserved #: bool
       attr_reader :precedence #: precedence?
       attr_reader :location #: location?
+      attr_reader :display_name #: String?
+      attr_reader :semantic_type #: String?
+      attr_reader :documentation #: String?
 
       # @rbs (id: Integer, name: String, kind: Symbol, ?reserved: bool, ?precedence: precedence?,
-      #   ?location: location?) -> void
-      def initialize(id:, name:, kind:, reserved: false, precedence: nil, location: nil)
+      #   ?location: location?, ?display_name: String?, ?semantic_type: String?, ?documentation: String?) -> void
+      def initialize(id:, name:, kind:, reserved: false, precedence: nil, location: nil, display_name: nil,
+                     semantic_type: nil, documentation: nil)
         @id = id
         @name = name.freeze
         @kind = kind.to_sym
         @reserved = reserved
         @precedence = IR.deep_freeze(precedence)
         @location = IR.deep_freeze(location)
+        @display_name = display_name&.freeze
+        @semantic_type = semantic_type&.freeze
+        @documentation = documentation&.freeze
         freeze
       end
 
@@ -44,9 +52,14 @@ module Ibex
       # @rbs () -> bool
       def nonterminal? = @kind == :nonterminal
 
-      # @rbs () -> Hash[Symbol, untyped]
-      def to_h
-        { id: @id, name: @name, kind: @kind, reserved: @reserved, prec: @precedence, loc: @location }
+      # @rbs (?schema_version: Integer) -> Hash[Symbol, untyped]
+      def to_h(schema_version: SCHEMA_VERSION)
+        value = { id: @id, name: @name, kind: @kind, reserved: @reserved,
+                  prec: @precedence, loc: @location } #: Hash[Symbol, untyped]
+        value[:display_name] = @display_name if @display_name
+        value[:semantic_type] = @semantic_type if @semantic_type
+        value[:doc] = @documentation if schema_version >= 2
+        value
       end
     end
 
@@ -56,19 +69,25 @@ module Ibex
       attr_reader :location #: location
       attr_reader :named_refs #: Array[named_ref]
       attr_reader :context_length #: Integer
+      attr_reader :composition #: action_composition?
 
-      # @rbs (code: String, location: location, ?named_refs: Array[named_ref], ?context_length: Integer) -> void
-      def initialize(code:, location:, named_refs: [], context_length: 0)
+      # @rbs (code: String, location: location, ?named_refs: Array[named_ref], ?context_length: Integer,
+      #   ?composition: action_composition?) -> void
+      def initialize(code:, location:, named_refs: [], context_length: 0, composition: nil)
         @code = code.freeze
         @location = IR.deep_freeze(location)
         @named_refs = IR.deep_freeze(named_refs)
         @context_length = context_length
+        @composition = IR.deep_freeze(composition)
         freeze
       end
 
-      # @rbs () -> Hash[Symbol, untyped]
-      def to_h
-        { code: @code, loc: @location, named_refs: @named_refs, context_length: @context_length }
+      # @rbs (?schema_version: Integer) -> Hash[Symbol, untyped]
+      def to_h(schema_version: SCHEMA_VERSION)
+        value = { code: @code, loc: @location, named_refs: @named_refs,
+                  context_length: @context_length } #: Hash[Symbol, untyped]
+        value[:composition] = @composition if schema_version >= 2
+        value
       end
     end
 
@@ -80,23 +99,37 @@ module Ibex
       attr_reader :action #: Action?
       attr_reader :precedence_override #: Integer?
       attr_reader :origin #: Hash[Symbol, untyped]
+      attr_reader :documentation #: String?
+      attr_reader :expansion #: production_expansion?
+      attr_reader :node #: node_annotation?
 
       # @rbs (id: Integer, lhs: Integer, rhs: Array[Integer], action: Action?, precedence_override: Integer?,
-      #   origin: Hash[Symbol, untyped]) -> void
-      def initialize(id:, lhs:, rhs:, action:, precedence_override:, origin:)
+      #   origin: Hash[Symbol, untyped], ?documentation: String?, ?expansion: production_expansion?,
+      #   ?node: node_annotation?) -> void
+      def initialize(id:, lhs:, rhs:, action:, precedence_override:, origin:, documentation: nil, expansion: nil,
+                     node: nil)
         @id = id
         @lhs = lhs
         @rhs = rhs.freeze
         @action = action
         @precedence_override = precedence_override
         @origin = IR.deep_freeze(origin)
+        @documentation = documentation&.freeze
+        @expansion = IR.deep_freeze(expansion)
+        @node = IR.deep_freeze(node)
         freeze
       end
 
-      # @rbs () -> Hash[Symbol, untyped]
-      def to_h
-        { id: @id, lhs: @lhs, rhs: @rhs, action: @action&.to_h, prec_override: @precedence_override,
-          origin: @origin }
+      # @rbs (?schema_version: Integer) -> Hash[Symbol, untyped]
+      def to_h(schema_version: SCHEMA_VERSION)
+        value = { id: @id, lhs: @lhs, rhs: @rhs, action: @action&.to_h(schema_version: schema_version),
+                  prec_override: @precedence_override, origin: @origin } #: Hash[Symbol, untyped]
+        if schema_version >= 2
+          value[:doc] = @documentation
+          value[:expansion] = @expansion
+          value[:node] = @node if @node
+        end
+        value
       end
     end
 
@@ -123,7 +156,15 @@ module Ibex
       attr_reader :class_name #: String
       attr_reader :superclass #: String?
       attr_reader :start #: String
+      attr_reader :starts #: Array[String]
+      attr_reader :mode #: grammar_mode
       attr_reader :expect #: Integer
+      attr_reader :expect_rr #: Integer?
+      attr_reader :parser_parameters #: Array[parser_parameter]
+      attr_reader :value_printers #: Array[value_printer]
+      attr_reader :grammar_tests #: Array[grammar_test]
+      attr_reader :lexer #: Lexer?
+      attr_reader :recovery #: recovery_policy
       attr_reader :options #: grammar_options
       attr_reader :symbols #: Array[GrammarSymbol]
       attr_reader :productions #: Array[Production]
@@ -132,17 +173,39 @@ module Ibex
       attr_reader :conversions #: Hash[String, String]
       attr_reader :warnings #: Array[grammar_warning]
       attr_reader :schema_version #: Integer
+      attr_reader :source_provenance #: source_provenance?
+      attr_reader :migration #: migration_metadata?
 
-      # @rbs (class_name: String, superclass: String?, start: String, expect: Integer, options: grammar_options,
+      # @rbs (class_name: String, superclass: String?, start: String, expect: Integer, ?expect_rr: Integer?,
+      #   options: grammar_options,
       #   symbols: Array[GrammarSymbol], productions: Array[Production], user_code: Hash[String, String],
       #   conversions: Hash[String, String], warnings: Array[grammar_warning], ?user_code_chunks: user_code_chunks?,
-      #   ?schema_version: Integer) -> void
+      #   ?schema_version: Integer, ?source_provenance: source_provenance?,
+      #   ?migration: migration_metadata?, ?parser_parameters: Array[parser_parameter],
+      #   ?value_printers: Array[value_printer], ?grammar_tests: Array[grammar_test],
+      #   ?recovery: recovery_policy?, ?lexer: Lexer?,
+      #   ?mode: grammar_mode, ?starts: Array[String]?) -> void
+      # rubocop:disable Metrics/AbcSize, Metrics/ParameterLists
+      # Immutable versioned IR is constructed from explicit public fields.
       def initialize(class_name:, superclass:, start:, expect:, options:, symbols:, productions:, user_code:,
-                     conversions:, warnings:, user_code_chunks: nil, schema_version: SCHEMA_VERSION)
+                     conversions:, warnings:, user_code_chunks: nil, schema_version: SCHEMA_VERSION,
+                     source_provenance: nil, migration: nil, expect_rr: nil, parser_parameters: [], value_printers: [],
+                     grammar_tests: [], recovery: nil, lexer: nil, mode: :default, starts: nil)
+        validate_mode(mode)
+        normalized_starts = validate_starts(start, starts, mode)
+
         @class_name = class_name.freeze
         @superclass = superclass&.freeze
         @start = start.freeze
+        @starts = normalized_starts.map(&:freeze).freeze
+        @mode = mode
         @expect = expect
+        @expect_rr = expect_rr
+        @parser_parameters = IR.deep_freeze(parser_parameters)
+        @value_printers = IR.deep_freeze(value_printers)
+        @grammar_tests = IR.deep_freeze(grammar_tests)
+        @lexer = lexer
+        @recovery = IR.deep_freeze(recovery || { sync_tokens: [], on_error_reduce: [] })
         @options = IR.deep_freeze(options)
         @symbols = symbols.freeze
         @productions = productions.freeze
@@ -152,10 +215,13 @@ module Ibex
         @conversions = IR.deep_freeze(conversions)
         @warnings = IR.deep_freeze(warnings)
         @schema_version = schema_version
+        @source_provenance = IR.deep_freeze(source_provenance)
+        @migration = IR.deep_freeze(migration)
         @symbols_by_name = @symbols.to_h { |symbol| [symbol.name, symbol] }.freeze
         @symbols_by_id = @symbols.to_h { |symbol| [symbol.id, symbol] }.freeze
         freeze
       end
+      # rubocop:enable Metrics/AbcSize, Metrics/ParameterLists
 
       # @rbs (String name) -> GrammarSymbol?
       def symbol(name) = @symbols_by_name[name]
@@ -169,16 +235,63 @@ module Ibex
       # @rbs () -> Hash[Symbol, untyped]
       def to_h
         value = { ibex_ir: "grammar", schema_version: @schema_version, class_name: @class_name, superclass: @superclass,
-                  start: @start, expect: @expect, options: @options, symbols: @symbols.map(&:to_h),
-                  productions: @productions.map(&:to_h), user_code: @user_code, conversions: @conversions,
+                  start: @start, expect: @expect, options: @options,
+                  symbols: @symbols.map { |symbol| symbol.to_h(schema_version: @schema_version) },
+                  productions: @productions.map { |production| production.to_h(schema_version: @schema_version) },
+                  user_code: @user_code, conversions: @conversions,
                   warnings: @warnings } #: Hash[Symbol, untyped]
-        value[:user_code_chunks] = @user_code_chunks.transform_values { |chunks| chunks.map(&:to_h) } \
-          unless @user_code_chunks.empty?
+        append_optional_metadata(value)
         value
       end
 
       private
 
+      # @rbs (grammar_mode mode) -> void
+      def validate_mode(mode)
+        raise ArgumentError, "mode must be :default or :extended" unless %i[default extended].include?(mode)
+      end
+
+      # @rbs (String start, Array[String]? starts, grammar_mode mode) -> Array[String]
+      def validate_starts(start, starts, mode)
+        values = starts || [start]
+        raise ArgumentError, "starts must contain the primary start symbol" if values.empty?
+        raise ArgumentError, "start must be the first entry in starts" unless values.first == start
+        raise ArgumentError, "starts must be unique" unless values.uniq.length == values.length
+        raise ArgumentError, "multiple start symbols require extended mode" if values.length > 1 && mode != :extended
+
+        values
+      end
+
+      # @rbs (Hash[Symbol, untyped] value) -> void
+      def append_optional_metadata(value)
+        append_parser_metadata(value)
+        append_recovery_metadata(value)
+        value[:user_code_chunks] = @user_code_chunks.transform_values { |chunks| chunks.map(&:to_h) } \
+          unless @user_code_chunks.empty?
+        return unless @schema_version >= 2
+
+        value[:source_provenance] = @source_provenance
+        value[:migration] = @migration
+      end
+
+      # @rbs (Hash[Symbol, untyped] value) -> void
+      def append_parser_metadata(value)
+        value[:expect_rr] = @expect_rr unless @expect_rr.nil?
+        value[:mode] = @mode if @mode == :extended
+        value[:starts] = @starts if @starts.length > 1
+        value[:params] = @parser_parameters unless @parser_parameters.empty?
+        value[:printers] = @value_printers unless @value_printers.empty?
+        value[:tests] = @grammar_tests unless @grammar_tests.empty? || @schema_version < 2
+        value[:lexer] = @lexer.to_h if @lexer && @schema_version >= 2
+      end
+
+      # @rbs (Hash[Symbol, untyped] value) -> void
+      def append_recovery_metadata(value)
+        return if @recovery[:sync_tokens].empty? && @recovery[:on_error_reduce].empty?
+
+        value[:recovery] = @recovery
+      end
+
       # @rbs () -> void
       def validate_user_code_chunks
         @user_code_chunks.each do |name, chunks|

data/lib/ibex/ir/lexer_ir.rb

@@ -0,0 +1,76 @@
+# frozen_string_literal: true
+
+module Ibex
+  module IR
+    LEXER_SCHEMA_VERSION = 1
+    SUPPORTED_LEXER_SCHEMA_VERSIONS = [LEXER_SCHEMA_VERSION].freeze #: Array[Integer]
+
+    # One normalized, state-scoped lexer rule.
+    class LexerRule
+      attr_reader :id #: Integer
+      attr_reader :state #: String
+      attr_reader :kind #: Symbol
+      attr_reader :token #: String?
+      attr_reader :pattern #: String
+      attr_reader :pattern_kind #: Symbol
+      attr_reader :options #: String
+      attr_reader :action #: String?
+      attr_reader :location #: location
+
+      # @rbs (id: Integer, state: String, kind: Symbol, token: String?, pattern: String, pattern_kind: Symbol,
+      #   options: String, action: String?, location: location) -> void
+      def initialize(id:, state:, kind:, token:, pattern:, pattern_kind:, options:, action:, location:)
+        @id = id
+        @state = state.freeze
+        @kind = kind
+        @token = token&.freeze
+        @pattern = pattern.freeze
+        @pattern_kind = pattern_kind
+        @options = options.freeze
+        @action = action&.freeze
+        @location = IR.deep_freeze(location)
+        freeze
+      end
+
+      # @rbs () -> Hash[Symbol, untyped]
+      def to_h
+        {
+          id: @id, state: @state, kind: @kind, token: @token, pattern: @pattern,
+          pattern_kind: @pattern_kind, options: @options, action: @action, loc: @location
+        }
+      end
+    end
+
+    # Independently versioned lexer contract embedded by Grammar IR v2.
+    class Lexer
+      attr_reader :states #: Array[String]
+      attr_reader :rules #: Array[LexerRule]
+      attr_reader :warnings #: Array[Hash[Symbol, untyped]]
+      attr_reader :schema_version #: Integer
+      attr_reader :source_provenance #: source_provenance?
+
+      # @rbs (states: Array[String], rules: Array[LexerRule], warnings: Array[Hash[Symbol, untyped]],
+      #   ?schema_version: Integer, ?source_provenance: source_provenance?) -> void
+      def initialize(states:, rules:, warnings:, schema_version: LEXER_SCHEMA_VERSION, source_provenance: nil)
+        raise ArgumentError, "lexer states must start with INITIAL" unless states.first == "INITIAL"
+        raise ArgumentError, "lexer states must be unique" unless states.uniq.length == states.length
+
+        @states = states.map(&:freeze).freeze
+        @rules = rules.freeze
+        @warnings = IR.deep_freeze(warnings)
+        @schema_version = schema_version
+        @source_provenance = IR.deep_freeze(source_provenance)
+        freeze
+      end
+
+      # @rbs () -> Hash[Symbol, untyped]
+      def to_h
+        {
+          ibex_ir: "lexer", schema_version: @schema_version, initial_state: "INITIAL",
+          states: @states, rules: @rules.map(&:to_h), warnings: @warnings,
+          source_provenance: @source_provenance
+        }
+      end
+    end
+  end
+end

data/lib/ibex/ir/migration.rb

@@ -0,0 +1,120 @@
+# frozen_string_literal: true
+
+module Ibex
+  module IR
+    # Meaning-preserving upgrades between published IR schema versions.
+    module Migration
+      UNAVAILABLE_V1_METADATA = %w[
+        source_provenance
+        symbol_docs
+        production_docs
+        production_expansion
+        action_composition
+        grammar_tests
+        lexer
+        cst
+        ast_nodes
+      ].freeze #: Array[String]
+
+      # @rbs (Grammar | Automaton value, ?to: Integer) -> (Grammar | Automaton)
+      def to_version(value, to: SCHEMA_VERSION)
+        return value if value.schema_version == to
+        return to_v2(value) if value.schema_version == 1 && to == 2
+
+        raise Ibex::Error,
+              "(ir):1:1: cannot migrate schema_version #{value.schema_version} to #{to}; only 1 to 2 is supported"
+      end
+      module_function :to_version
+
+      # @rbs (Grammar | Automaton value) -> (Grammar | Automaton)
+      def to_v2(value)
+        return value if value.schema_version == 2
+        unless value.schema_version == 1
+          raise Ibex::Error, "(ir):1:1: cannot migrate unsupported schema_version #{value.schema_version}"
+        end
+
+        return migrate_grammar(value) if value.is_a?(Grammar)
+        return migrate_automaton(value) if value.is_a?(Automaton)
+
+        raise Ibex::Error, "(ir):1:1: cannot migrate unsupported IR object #{value.class}"
+      end
+      module_function :to_v2
+
+      # @rbs (Grammar grammar) -> Grammar
+      def migrate_grammar(grammar)
+        symbols = migrate_symbols(grammar)
+        productions = migrate_productions(grammar)
+        build_migrated_grammar(grammar, symbols, productions)
+      end
+      module_function :migrate_grammar
+
+      # @rbs (Grammar grammar) -> Array[GrammarSymbol]
+      def migrate_symbols(grammar)
+        grammar.symbols.map do |symbol|
+          GrammarSymbol.new(
+            id: symbol.id, name: symbol.name, kind: symbol.kind, reserved: symbol.reserved,
+            precedence: symbol.precedence, location: symbol.location, display_name: symbol.display_name,
+            semantic_type: symbol.semantic_type
+          )
+        end
+      end
+      module_function :migrate_symbols
+
+      # @rbs (Grammar grammar) -> Array[Production]
+      def migrate_productions(grammar)
+        grammar.productions.map do |production|
+          Production.new(
+            id: production.id, lhs: production.lhs, rhs: production.rhs,
+            action: migrate_action(production.action), precedence_override: production.precedence_override,
+            origin: production.origin
+          )
+        end
+      end
+      module_function :migrate_productions
+
+      # @rbs (Grammar grammar, Array[GrammarSymbol] symbols, Array[Production] productions) -> Grammar
+      def build_migrated_grammar(grammar, symbols, productions)
+        Grammar.new(
+          class_name: grammar.class_name, superclass: grammar.superclass, start: grammar.start,
+          mode: grammar.mode, starts: grammar.starts,
+          expect: grammar.expect, expect_rr: grammar.expect_rr, options: grammar.options,
+          parser_parameters: grammar.parser_parameters,
+          value_printers: grammar.value_printers,
+          grammar_tests: grammar.grammar_tests,
+          recovery: grammar.recovery,
+          symbols: symbols, productions: productions,
+          user_code: grammar.user_code, user_code_chunks: grammar.user_code_chunks,
+          conversions: grammar.conversions, warnings: grammar.warnings, schema_version: 2,
+          migration: { from_schema_version: 1, unavailable: UNAVAILABLE_V1_METADATA }
+        )
+      end
+      module_function :build_migrated_grammar
+
+      # @rbs (Action? action) -> Action?
+      def migrate_action(action)
+        return nil unless action
+
+        Action.new(
+          code: action.code, location: action.location, named_refs: action.named_refs,
+          context_length: action.context_length
+        )
+      end
+      module_function :migrate_action
+
+      # @rbs (Automaton automaton) -> Automaton
+      def migrate_automaton(automaton)
+        Automaton.new(
+          grammar: migrate_grammar(automaton.grammar), states: automaton.states,
+          conflict_summary: automaton.conflict_summary, algorithm: automaton.algorithm, schema_version: 2,
+          entry_states: automaton.entry_states
+        )
+      end
+      module_function :migrate_automaton
+
+      class << self
+        private :migrate_grammar, :migrate_symbols, :migrate_productions, :build_migrated_grammar,
+                :migrate_action, :migrate_automaton
+      end
+    end
+  end
+end