ibex-runtime 0.2.0

data/lib/ibex/runtime/cst/incremental/session.rb

@@ -0,0 +1,229 @@
+# frozen_string_literal: true
+# rbs_inline: enabled
+
+module Ibex
+  module Runtime
+    module CST
+      # Raised when incremental parsing cannot preserve its syntax-only contract.
+      class IncrementalUnsupportedError < ArgumentError; end
+
+      # A mutable, single-owner session around immutable source and syntax results.
+      class IncrementalParseSession
+        attr_reader :source_text #: SourceText
+        attr_reader :result #: SyntaxResult
+        attr_reader :token_memo #: TokenMemo
+        attr_reader :parse_memo #: ParseMemo
+        attr_reader :last_relex_result #: RelexResult?
+        attr_reader :last_blender #: Blender?
+
+        # @rbs @resource_limits: ResourceLimits
+        # @rbs @parser: Parser
+        # @rbs @cache: NodeCache
+        # @rbs @blender_enabled: bool
+        # @rbs @mutex: Mutex
+
+        # @rbs (Class parser_class, SourceText source_text, ?resource_limits: ResourceLimits?, ?blender: bool) -> void
+        def initialize(parser_class, source_text, resource_limits: nil, blender: true)
+          unless source_text.is_a?(SourceText)
+            raise ArgumentError, "incremental_session requires an Ibex::Runtime::CST::SourceText"
+          end
+
+          @resource_limits = resource_limits || ResourceLimits.new #: ResourceLimits
+          @parser = parser_class.__send__(:new, resource_limits: @resource_limits) #: Parser
+          validate_parser!
+          @cache = NodeCache.new #: NodeCache
+          @source_text = source_text
+          @last_relex_result = nil #: RelexResult?
+          @last_blender = nil #: Blender?
+          @blender_enabled = blender
+          @mutex = Mutex.new
+          @result, @token_memo, @parse_memo = parse_current(0.0)
+          enforce_memo_budget!
+        end
+
+        # Apply edits expressed against the current source and return syntax only.
+        # @rbs (Array[TextEdit] edits) -> SyntaxResult
+        def edit(edits)
+          @mutex.synchronize { edit_locked(TextEdit.normalize(edits)) }
+        end
+
+        # Observe parser and incremental runtime events.
+        # @rbs () { (Event) -> void } -> Observation::Subscription
+        def observe(&observer)
+          @parser.observe(&observer)
+        end
+
+        # @rbs (Observation::Subscription subscription) -> bool
+        def unobserve(subscription)
+          @parser.unobserve(subscription)
+        end
+
+        private
+
+        # @rbs (Array[TextEdit] edits) -> SyntaxResult
+        def edit_locked(edits)
+          return @result if edits.empty?
+
+          previous_memo = @token_memo
+          previous_root = @result.syntax_root
+          previous_parse_memo = @parse_memo
+          @source_text = @source_text.apply(edits)
+          lexed = scan_current
+          return finish_full_fallback(previous_memo, edits, :lexical_error) unless lexed
+
+          relexed = Relexer.reconcile(previous_memo, lexed.memo, edits)
+          blender = build_blender(previous_root, previous_parse_memo, lexed.with_memo(relexed.memo), edits)
+          finish_blended_edit(relexed, blender)
+        end
+
+        # @rbs () -> LexedSyntax?
+        def scan_current
+          @parser.__send__(:scan_syntax_with_cache, @source_text, @cache)
+        rescue ParseError
+          nil
+        end
+
+        # @rbs (SyntaxNode old_root, ParseMemo old_memo, LexedSyntax lexed, Array[TextEdit] edits) -> Blender
+        def build_blender(old_root, old_memo, lexed, edits)
+          Blender.new(
+            old_root: old_root,
+            parse_memo: old_memo,
+            lexed: lexed,
+            edits: edits,
+            max_decomposed_nodes: @resource_limits.max_incremental_decomposed_nodes,
+            enabled: @blender_enabled
+          )
+        end
+
+        # @rbs (RelexResult relexed, Blender blender) -> SyntaxResult
+        def finish_blended_edit(relexed, blender)
+          @last_relex_result = relexed
+          @last_blender = blender
+          parsed = @parser.__send__(:parse_syntax_token_source, blender, @cache)
+          reused_ratio = blender_reused_ratio(blender, parsed.syntax_root.green, relexed)
+          @token_memo = relexed.memo
+          @parse_memo = @parser.__send__(:syntax_parse_memo) || empty_parse_memo(parsed.syntax_root.green)
+          @result = SyntaxResult.new(
+            syntax_root: parsed.syntax_root,
+            diagnostics: parsed.diagnostics,
+            reused_ratio: reused_ratio
+          )
+          report_blender_fallback(blender)
+          enforce_memo_budget!
+          emit_reuse_event(blender, relexed, reused_ratio)
+          @result
+        end
+
+        # @rbs (Blender blender, GreenNode root, RelexResult relexed) -> Float
+        def blender_reused_ratio(blender, root, relexed)
+          return relexed.reused_ratio unless @blender_enabled
+
+          blender.reused_descendants.fdiv(root.descendant_count)
+        end
+
+        # @rbs (Blender blender, RelexResult relexed, Float reused_ratio) -> void
+        def emit_reuse_event(blender, relexed, reused_ratio)
+          @parser.__send__(
+            :emit_incremental_event,
+            :cst_reuse,
+            {
+              "stage" => blender.reused_descendants.positive? ? "subtree" : "lexical",
+              "reused_tokens" => relexed.reused_count,
+              "token_count" => relexed.memo.tokens.length,
+              "reused_ratio" => reused_ratio
+            }
+          )
+        end
+
+        # @rbs () -> void
+        def validate_parser!
+          unless @parser.is_a?(GeneratedLexer)
+            raise IncrementalUnsupportedError, "incremental parsing requires a generated lexer"
+          end
+
+          tables = @parser.__send__(:parser_tables) #: Hash[Symbol, untyped]
+          config = tables[:cst]
+          unless tables.fetch(:format_version) >= 6 && config.is_a?(Hash)
+            raise IncrementalUnsupportedError, "incremental parsing requires a format-v6 Red/Green CST parser"
+          end
+          return unless config.fetch(:trivia_policy) == :drop
+
+          raise IncrementalUnsupportedError, "incremental parsing does not support drop trivia policy"
+        end
+
+        # @rbs (Float reused_ratio) -> [SyntaxResult, TokenMemo, ParseMemo]
+        def parse_current(reused_ratio)
+          parsed = @parser.__send__(:parse_syntax_with_cache, @source_text, @cache)
+          result = SyntaxResult.new(
+            syntax_root: parsed.syntax_root,
+            diagnostics: parsed.diagnostics,
+            reused_ratio: reused_ratio
+          )
+          states = @parser.__send__(:syntax_token_states)
+          parse_memo = @parser.__send__(:syntax_parse_memo) || empty_parse_memo(result.syntax_root.green)
+
+          [result, TokenMemo.from_root(result.syntax_root, states: states), parse_memo]
+        end
+
+        # Error trees cannot be reused, but retain position-aligned disposable memo storage.
+        # @rbs (GreenNode root) -> ParseMemo
+        def empty_parse_memo(root)
+          tables = @parser.__send__(:parser_tables) #: Hash[Symbol, untyped]
+          ParseMemo.new(
+            left_states: Array.new(root.descendant_count),
+            grammar_digest: tables.fetch(:grammar_digest),
+            state_count: tables.fetch(:state_count),
+            production_count: tables.fetch(:production_count)
+          )
+        end
+
+        # @rbs (Blender blender) -> void
+        def report_blender_fallback(blender)
+          return unless blender.fallback_reason
+
+          @parser.__send__(
+            :emit_incremental_event,
+            :cst_fallback,
+            {
+              "reason" => blender.fallback_reason.to_s,
+              "limit" => @resource_limits.max_incremental_decomposed_nodes,
+              "observed" => blender.decomposed_nodes
+            }
+          )
+        end
+
+        # @rbs (TokenMemo previous_memo, Array[TextEdit] edits, Symbol reason) -> SyntaxResult
+        def finish_full_fallback(previous_memo, edits, reason)
+          fresh_result, fresh_memo, fresh_parse_memo = parse_current(0.0)
+          @last_relex_result = Relexer.reconcile(previous_memo, fresh_memo, edits)
+          @last_blender = nil
+          @token_memo = fresh_memo
+          @parse_memo = fresh_parse_memo
+          @result = fresh_result
+          @parser.__send__(
+            :emit_incremental_event,
+            :cst_fallback,
+            { "reason" => reason.to_s, "limit" => 0, "observed" => 0 }
+          )
+          enforce_memo_budget!
+          @result
+        end
+
+        # @rbs () -> void
+        def enforce_memo_budget!
+          observed = @token_memo.estimated_bytes + @parse_memo.estimated_bytes
+          limit = @resource_limits.max_session_memo_bytes
+          return unless observed > limit
+
+          @parser.__send__(
+            :emit_incremental_event,
+            :cst_fallback,
+            { "reason" => "memo_budget", "limit" => limit, "observed" => observed }
+          )
+          @cache.clear
+          @token_memo = TokenMemo.from_root(@result.syntax_root)
+        end
+      end
+    end
+  end
+end

data/lib/ibex/runtime/cst/incremental/token_memo.rb

@@ -0,0 +1,66 @@
+# frozen_string_literal: true
+# rbs_inline: enabled
+
+module Ibex
+  module Runtime
+    module CST
+      # Session-local token metadata used to validate lexical resynchronization.
+      class TokenMemo
+        ENTRY_BYTES = 32 #: Integer
+
+        attr_reader :tokens #: Array[GreenToken]
+        attr_reader :offsets #: Array[Integer]
+        attr_reader :states #: Array[Symbol]
+
+        # @rbs (tokens: Array[GreenToken], offsets: Array[Integer], states: Array[Symbol]) -> void
+        def initialize(tokens:, offsets:, states:)
+          unless tokens.length == offsets.length && offsets.length == states.length
+            raise ArgumentError, "token memo arrays must have equal lengths"
+          end
+
+          ordered = (0...[offsets.length - 1, 0].max).all? do |index|
+            offsets.fetch(index) <= offsets.fetch(index + 1)
+          end
+          raise ArgumentError, "token memo offsets must be ordered" unless ordered
+
+          @tokens = tokens.dup.freeze
+          @offsets = offsets.dup.freeze
+          @states = states.dup.freeze
+          freeze
+        end
+
+        # @rbs (SyntaxNode root, ?states: Array[Symbol]) -> TokenMemo
+        def self.from_root(root, states: [])
+          tokens = root.tokens.map(&:green)
+          offsets = [] #: Array[Integer]
+          offset = 0
+          tokens.each do |token|
+            offsets << offset
+            offset += token.full_width
+          end
+          actual_states = Array.new(tokens.length) { |index| states.fetch(index, :INITIAL) }
+          new(tokens: tokens, offsets: offsets, states: actual_states)
+        end
+
+        # The first token whose owned full span contains or follows an edit.
+        # @rbs (Array[TextEdit] edits) -> Integer?
+        def damage_index(edits)
+          normalized = TextEdit.normalize(edits)
+          edit = normalized.first
+          return unless edit
+
+          point = edit.start
+          @tokens.each_index do |index|
+            start = @offsets.fetch(index)
+            finish = start + @tokens.fetch(index).full_width
+            return index if point < finish || (start == point && finish == point)
+          end
+          @tokens.empty? ? nil : @tokens.length - 1
+        end
+
+        # @rbs () -> Integer
+        def estimated_bytes = @tokens.length * ENTRY_BYTES
+      end
+    end
+  end
+end

data/lib/ibex/runtime/cst/kind.rb

@@ -0,0 +1,133 @@
+# frozen_string_literal: true
+# rbs_inline: enabled
+
+module Ibex
+  module Runtime
+    module CST
+      # Flags cached on Green elements and propagated by Green nodes.
+      module Flags
+        CONTAINS_ERROR = 1 << 0 #: Integer
+        CONTAINS_MISSING = 1 << 1 #: Integer
+        CONTAINS_SKIPPED = 1 << 2 #: Integer
+        HAS_ANNOTATION = 1 << 3 #: Integer
+        SYNTHETIC = 1 << 4 #: Integer
+        INCOMPLETE_INPUT = 1 << 5 #: Integer
+      end
+
+      # Grammar-specific integer kind names and interval predicates.
+      class Kind
+        # @rbs!
+        #   type kind_map = Hash[String, Integer]
+        #   type field_slot = Integer | { index: Integer, extraction: Symbol }
+        #   type slot = { node_kind: Integer, node_name: String, fields: Hash[String, field_slot] }
+        #   type metadata = {
+        #     names: Array[String],
+        #     terminal_range: Array[Integer],
+        #     nonterminal_range: Array[Integer],
+        #     named: kind_map,
+        #     named_nonterminals: Hash[Integer, Integer],
+        #     trivia: kind_map,
+        #     synthetic: kind_map
+        #   }
+
+        # @rbs @names: Array[String]
+        # @rbs @terminal_range: Array[Integer]
+        # @rbs @nonterminal_range: Array[Integer]
+        # @rbs @named: Hash[String, Integer]
+        # @rbs @named_nonterminals: Hash[Integer, Integer]
+        # @rbs @trivia: Hash[String, Integer]
+        # @rbs @synthetic: Hash[String, Integer]
+        # @rbs @field_slots: Hash[Integer, Hash[String, field_slot]]
+
+        # @rbs (metadata metadata, ?slots: Hash[Integer, slot]) -> void
+        def initialize(metadata, slots: {})
+          @names = metadata.fetch(:names).dup.freeze
+          @terminal_range = metadata.fetch(:terminal_range).dup.freeze
+          @nonterminal_range = metadata.fetch(:nonterminal_range).dup.freeze
+          @named = metadata.fetch(:named).dup.freeze
+          @named_nonterminals = metadata.fetch(:named_nonterminals).dup.freeze
+          @trivia = metadata.fetch(:trivia).dup.freeze
+          @synthetic = metadata.fetch(:synthetic).dup.freeze
+          @field_slots = build_field_slots(slots)
+          freeze
+        end
+
+        # @rbs (Integer kind) -> String
+        def name(kind)
+          @names.fetch(kind)
+        end
+
+        # @rbs (String | Symbol name) -> Integer
+        def fetch(name)
+          key = name.to_s
+          @named[key] || @trivia[key] || @synthetic.fetch(key)
+        end
+
+        # @rbs (Integer kind) -> bool
+        def terminal?(kind) = within?(@terminal_range, kind)
+
+        # @rbs (Integer kind) -> bool
+        def nonterminal?(kind)
+          within?(@nonterminal_range, kind) || @named_nonterminals.key?(kind)
+        end
+
+        # @rbs (Integer kind) -> bool
+        def trivia?(kind) = @trivia.value?(kind)
+
+        # @rbs (Integer kind) -> bool
+        def error?(kind)
+          kind == @synthetic["lexical_error_token"] || kind == @synthetic["error_node"]
+        end
+
+        # Map a named node kind to its physical nonterminal kind.
+        # @rbs (Integer kind) -> Integer
+        def nonterminal_of(kind)
+          @named_nonterminals.fetch(kind, kind)
+        end
+
+        # Return the named field-to-physical-slot mapping for a kind.
+        # @rbs (Integer kind) -> Hash[String, field_slot]
+        def fields(kind) = @field_slots.fetch(kind, EMPTY_FIELDS)
+
+        # Return the serialization-safe kind metadata.
+        # @rbs () -> metadata
+        def to_h
+          value = {
+            names: @names, terminal_range: @terminal_range, nonterminal_range: @nonterminal_range,
+            named: @named, named_nonterminals: @named_nonterminals,
+            trivia: @trivia, synthetic: @synthetic
+          } #: metadata
+          value.freeze
+        end
+
+        private
+
+        empty_fields = {} #: Hash[String, field_slot]
+        EMPTY_FIELDS = empty_fields.freeze
+        private_constant :EMPTY_FIELDS
+
+        # @rbs (Hash[Integer, slot] slots) -> Hash[Integer, Hash[String, field_slot]]
+        def build_field_slots(slots)
+          values = {} #: Hash[Integer, Hash[String, field_slot]]
+          slots.each_value do |slot|
+            kind = slot.fetch(:node_kind)
+            fields = slot.fetch(:fields).to_h do |name, field_slot|
+              index = field_slot.is_a?(Hash) ? field_slot.fetch(:index) : field_slot
+              [name, index]
+            end.freeze
+            previous = values[kind]
+            raise ArgumentError, "conflicting CST fields for kind #{kind}" if previous && previous != fields
+
+            values[kind] = fields
+          end
+          values.freeze
+        end
+
+        # @rbs (Array[Integer] range, Integer kind) -> bool
+        def within?(range, kind)
+          kind >= range.fetch(0) && kind < range.fetch(1)
+        end
+      end
+    end
+  end
+end

data/lib/ibex/runtime/cst/parse_result.rb

@@ -0,0 +1,41 @@
+# frozen_string_literal: true
+# rbs_inline: enabled
+
+module Ibex
+  module Runtime
+    module CST
+      # Immutable result returned by syntax-aware parser entry points.
+      class ParseResult
+        attr_reader :value #: untyped
+        attr_reader :syntax_root #: SyntaxNode
+        attr_reader :diagnostics #: Array[untyped]
+
+        # @rbs (value: untyped, syntax_root: SyntaxNode, diagnostics: Array[untyped]) -> void
+        def initialize(value:, syntax_root:, diagnostics:)
+          @value = value
+          @syntax_root = syntax_root
+          @diagnostics = diagnostics.dup.freeze
+          freeze
+        end
+      end
+
+      # Syntax-only result used by incremental sessions.
+      class SyntaxResult
+        attr_reader :syntax_root #: SyntaxNode
+        attr_reader :diagnostics #: Array[untyped]
+        attr_reader :reused_ratio #: Float
+
+        # @rbs (syntax_root: SyntaxNode, diagnostics: Array[untyped], reused_ratio: Float) -> void
+        def initialize(syntax_root:, diagnostics:, reused_ratio:)
+          @syntax_root = syntax_root
+          @diagnostics = diagnostics.dup.freeze
+          @reused_ratio = reused_ratio
+          freeze
+        end
+      end
+    end
+
+    ParseResult = CST::ParseResult
+    SyntaxResult = CST::SyntaxResult
+  end
+end

data/lib/ibex/runtime/cst/rewriter.rb

@@ -0,0 +1,75 @@
+# frozen_string_literal: true
+# rbs_inline: enabled
+
+require_relative "editing" unless defined?(Ibex::Runtime::CST::Editing)
+
+module Ibex
+  module Runtime
+    module CST
+      # Bottom-up syntax rewriter with kind-name dispatch.
+      class SyntaxRewriter
+        # @rbs! type element = SyntaxNode | SyntaxToken
+        # @rbs! type green = GreenNode | GreenToken
+
+        # @rbs (SyntaxNode root) -> SyntaxNode
+        def rewrite(root)
+          green = visit(root)
+          return root if green.equal?(root.green)
+          raise TypeError, "a syntax root must remain a GreenNode" unless green.is_a?(GreenNode)
+
+          Editing.red_root(root, green)
+        end
+
+        # @rbs (element element) -> green
+        def visit(element)
+          rewritten = element.is_a?(SyntaxNode) ? visit_children(element) : element.green
+          method = :"visit_#{method_name(element.kind_name)}"
+          return rewritten unless respond_to?(method, true)
+
+          Editing.green_element(__send__(method, rewritten_view(element, rewritten)))
+        end
+
+        private
+
+        # @rbs (SyntaxNode node) -> GreenNode
+        def visit_children(node)
+          changed = false
+          children = node.children.map do |child|
+            green = visit(child)
+            changed ||= !green.equal?(child.green)
+            green
+          end
+          return node.green unless changed
+
+          GreenNode.new(
+            kind: node.green.kind, children: children,
+            flags: node.green.intrinsic_flags, annotations: node.green.annotations
+          )
+        end
+
+        # @rbs (element original, green rewritten) -> element
+        def rewritten_view(original, rewritten)
+          if rewritten.is_a?(GreenNode)
+            raise TypeError, "a node rewrite requires its original node" unless original.is_a?(SyntaxNode)
+
+            source = SourceText.new(rewritten.to_source, file: original.root.source_text.file)
+            return SyntaxNode.new(
+              green: rewritten, kinds: original.kinds,
+              trivia_policy: original.root.trivia_policy, source_text: source
+            )
+          end
+
+          parent = original.parent
+          raise TypeError, "a token rewrite requires its original parent" unless parent
+
+          SyntaxToken.new(green: rewritten, parent: parent, index: original.index, offset: original.offset)
+        end
+
+        # @rbs (String name) -> String
+        def method_name(name)
+          name.gsub(/[^a-zA-Z0-9_]/, "_")
+        end
+      end
+    end
+  end
+end

data/lib/ibex/runtime/cst/serialize.rb

@@ -0,0 +1,127 @@
+# frozen_string_literal: true
+# rbs_inline: enabled
+
+require "json"
+
+module Ibex
+  module Runtime
+    module CST
+      class SerializationError < StandardError; end
+
+      # Stable JSON serialization for `ibex_cst` schema version 1.
+      module Serialize
+        SCHEMA_VERSION = 1 #: Integer
+
+        # @rbs (SyntaxNode | SerializedTree value, ?grammar_digest: String?, ?table_format: Integer?,
+        #   ?state_count: Integer?, ?production_count: Integer?, ?memo: ParseMemo?) -> String
+        def dump(value, grammar_digest: nil, table_format: nil, state_count: nil, production_count: nil, memo: nil)
+          tree = serialization_tree(
+            value, grammar_digest: grammar_digest, table_format: table_format,
+                   state_count: state_count, production_count: production_count, memo: memo
+          )
+          document = {
+            "ibex_ir" => "cst",
+            "schema_version" => SCHEMA_VERSION,
+            "grammar_digest" => tree.grammar_digest,
+            "table_format" => tree.table_format,
+            "state_count" => tree.state_count,
+            "production_count" => tree.production_count,
+            "trivia_policy" => tree.trivia_policy.to_s,
+            "kinds" => kinds_document(tree.kinds),
+            "root" => element_document(tree.green_root),
+            "memo" => tree.memo&.to_h
+          }
+          "#{JSON.pretty_generate(document)}\n"
+        end
+        module_function :dump
+
+        # @rbs (String source, ?grammar_digest: String?, ?state_count: Integer?,
+        #   ?production_count: Integer?) -> SerializedTree
+        def load(source, grammar_digest: nil, state_count: nil, production_count: nil)
+          Validator.validate(
+            source,
+            grammar_digest: grammar_digest,
+            state_count: state_count,
+            production_count: production_count
+          )
+        end
+        module_function :load
+
+        # @rbs (SyntaxNode | SerializedTree value, grammar_digest: String?, table_format: Integer?,
+        #   state_count: Integer?, production_count: Integer?, memo: ParseMemo?) -> SerializedTree
+        def serialization_tree(value, grammar_digest:, table_format:, state_count:, production_count:, memo:)
+          return value if value.is_a?(SerializedTree)
+
+          missing = {
+            grammar_digest: grammar_digest, table_format: table_format,
+            state_count: state_count, production_count: production_count
+          }.select { |_name, item| item.nil? }.keys
+          raise ArgumentError, "missing CST serialization metadata: #{missing.join(', ')}" unless missing.empty?
+
+          SerializedTree.new(
+            grammar_digest: grammar_digest || "", table_format: table_format || 0,
+            state_count: state_count || 0, production_count: production_count || 0,
+            trivia_policy: value.trivia_policy, kinds: value.kinds, green_root: value.green, memo: memo
+          )
+        end
+        module_function :serialization_tree
+        private_class_method :serialization_tree
+
+        # @rbs (Kind kinds) -> Hash[String, untyped]
+        def kinds_document(kinds)
+          metadata = kinds.to_h
+          {
+            "names" => metadata.fetch(:names),
+            "terminal_range" => metadata.fetch(:terminal_range),
+            "nonterminal_range" => metadata.fetch(:nonterminal_range),
+            "named" => metadata.fetch(:named),
+            "named_nonterminals" => metadata.fetch(:named_nonterminals).to_h { |key, value| [key.to_s, value] },
+            "trivia" => metadata.fetch(:trivia),
+            "synthetic" => metadata.fetch(:synthetic)
+          }
+        end
+        module_function :kinds_document
+        private_class_method :kinds_document
+
+        # @rbs (GreenNode | GreenToken element) -> Hash[String, untyped]
+        def element_document(element)
+          if element.is_a?(GreenNode)
+            raise SerializationError, "annotated Green nodes cannot be serialized" unless element.annotations.empty?
+
+            return {
+              "k" => element.kind,
+              "f" => element.intrinsic_flags,
+              "c" => element.children.map { |child| element_document(child) }
+            }
+          end
+
+          {
+            "k" => element.kind,
+            "f" => element.flags,
+            "e" => element.expected_kind,
+            "t" => encode_text(element.text),
+            "l" => element.leading.map { |trivia| trivia_document(trivia) },
+            "r" => element.trailing.map { |trivia| trivia_document(trivia) }
+          }
+        end
+        module_function :element_document
+        private_class_method :element_document
+
+        # @rbs (GreenTrivia trivia) -> Array[untyped]
+        def trivia_document(trivia) = [trivia.kind, encode_text(trivia.text)]
+        module_function :trivia_document
+        private_class_method :trivia_document
+
+        # @rbs (String text) -> (String | Hash[String, String])
+        def encode_text(text)
+          utf8 = text.dup.force_encoding(Encoding::UTF_8)
+          return utf8 if utf8.valid_encoding?
+
+          { "b64" => [text.b].pack("m0") }
+        end
+        module_function :encode_text
+        private_class_method :encode_text
+      end
+    end
+  end
+end