flexr 1.0.0

data/lib/flexr/runtime/errors.rb

@@ -0,0 +1,22 @@
+# frozen_string_literal: true
+
+module Flexr
+  module Runtime
+    class TokenTooLargeError < LexError
+      CODE = "FLEXR-E012"
+
+      attr_reader :code
+
+      def initialize(message = "token exceeds max_token_size", filename: nil, byte_pos: nil, line: nil, text: nil)
+        @code = CODE
+        diagnostic = Diagnostics.error(
+          CODE,
+          message,
+          help: "increase max_token_size or split the input token"
+        )
+        super(message, filename: filename, byte_pos: byte_pos, line: line, text: text, diagnostic: diagnostic)
+      end
+    end
+    class StateStackOverflowError < LexError; end
+  end
+end

data/lib/flexr/runtime/interpreter.rb

@@ -0,0 +1,505 @@
+# frozen_string_literal: true
+
+begin
+  require "strscan"
+rescue LoadError
+  # StringScanner is an optional accelerator; the regexp path remains valid.
+end
+
+module Flexr
+  module Runtime
+    Match = Struct.new(:rule, :start_pos, :end_pos, :total_end_pos, keyword_init: true)
+
+    class Interpreter
+      def initialize(lexer)
+        @lexer = lexer
+      end
+
+      def scan
+        machine = @lexer.class.__flexr_compiled.machines.fetch(@lexer.state)
+        position = @lexer.byte_pos
+        return nil unless @lexer.valid_utf8_at?(position)
+        return scan_firstmatch(position) if @lexer.class.__flexr_config.backend == :firstmatch
+        return scan_fast(machine, position) if fast_path?
+
+        buffer = @lexer.buffer
+        state = machine.dfa.start
+        best = reference_match(position, buffer)
+        cursor = position
+        best = consider_acceptances(machine, state, cursor, position, buffer, best) if @lexer.class.__flexr_config.options[:allow_empty_match]
+        acceleration_regions = acceleration_enabled? ? acceleration_regions_for(machine) : nil
+
+        while buffer.ensure_available?(cursor + 1)
+          if acceleration_regions && (region = acceleration_regions[state])
+            accelerated_end = accelerate(region, buffer, cursor)
+            if accelerated_end && accelerated_end > cursor
+              cursor = accelerated_end
+              best = consider_acceptances(machine, state, cursor, position, buffer, best)
+              next
+            end
+          end
+
+          byte = buffer.getbyte(cursor)
+          state = transition(machine.dfa, state, byte)
+          break unless state
+
+          cursor += 1
+          best = consider_acceptances(machine, state, cursor, position, buffer, best)
+        end
+        best
+      end
+
+      def scan_fast(machine, position)
+        buffer = @lexer.buffer
+        source = buffer.source
+        dfa = machine.dfa
+        accepts = dfa.accepts
+        rules = @lexer.class.__flexr_rules
+        transitions = dfa.transitions
+        ec = dfa.ec
+        direct = dfa.direct
+        state = dfa.start
+        cursor = position
+        best = nil
+
+        while cursor < source.bytesize || buffer.ensure_available?(cursor + 1)
+          byte = source.getbyte(cursor)
+          state = if direct
+            class_id = ec[byte]
+            value = direct[:nxt][(state * direct[:classes]) + class_id]
+            value >= 0 ? value : nil
+          else
+            transitions[state][ec[byte]]
+          end
+          break unless state
+
+          cursor += 1
+          acceptance = accepts[state].first
+          next unless acceptance
+
+          rule = rules.fetch(acceptance.rule_index)
+          next if best && cursor == best.total_end_pos && rule.index > best.rule.index
+
+          best ||= (@match ||= Match.new)
+          best.rule = rule
+          best.start_pos = position
+          best.end_pos = cursor
+          best.total_end_pos = cursor
+        end
+        best
+      end
+
+      private
+
+      def reference_match(position, buffer)
+        return unless reference_rules?
+
+        candidates = @lexer.class.__flexr_rules.filter_map do |rule|
+          next unless rule_active?(rule)
+          next unless rule.patterns.any? { |pattern| reference_pattern?(pattern) }
+
+          matches = rule.patterns.each_with_index.filter_map do |pattern, pattern_index|
+            condition = rule.pattern_conditions.fetch(pattern_index)
+            next if condition.bol_only && !@lexer.beginning_of_line?
+
+            match = streamed_match(pattern, buffer, position, reference: reference_pattern?(pattern))
+            next unless match&.begin(0)&.zero?
+
+            end_position = position + match[0].bytesize
+            next unless @lexer.utf8_boundary?(end_position)
+            next if condition.end_anchor && !end_anchor_match?(buffer, end_position)
+
+            [match, condition]
+          end
+          match, _condition = matches.max_by { |item| item[0][0].bytesize }
+          next unless match&.begin(0)&.zero?
+
+          end_position = position + match[0].bytesize
+          next unless @lexer.utf8_boundary?(end_position)
+
+          trailing = trailing_length(rule, buffer, end_position)
+          next if rule.trailing && trailing.nil?
+
+          ensure_token_size!(end_position, position)
+          Match.new(rule: rule, start_pos: position, end_pos: end_position,
+                    total_end_pos: end_position + (trailing || 0))
+        end
+        candidates.max_by { |candidate| [candidate.total_end_pos, -candidate.rule.index] }
+      end
+
+      def reference_rules?
+        return @reference_rules unless @reference_rules.nil?
+
+        @reference_rules = @lexer.class.__flexr_rules.any? do |rule|
+          rule.patterns.any? { |pattern| reference_pattern?(pattern) }
+        end
+      end
+
+      def fast_path?
+        return @fast_path unless @fast_path.nil?
+
+        rules = @lexer.class.__flexr_rules
+        @fast_path = !@lexer.class.__flexr_config.options[:allow_empty_match] &&
+          !reference_rules? && rules.none?(&:trailing) &&
+          rules.none? { |rule| rule.pattern_conditions.any? { |condition| condition&.bol_only || condition&.end_anchor } }
+      end
+
+      def reference_pattern?(pattern)
+        return false unless pattern.is_a?(::Regexp)
+        return true if pattern.source.match?(/\\[pP]\{/) || pattern.source.match?(/\[:(?:\^)?[a-z]+:\]/)
+
+        @lexer.class.__flexr_config.options[:unicode] == true && @lexer.utf8_input? &&
+          pattern.source.match?(/\\[dDwWsS]/)
+      end
+
+      def scan_firstmatch(position)
+        buffer = @lexer.buffer
+        @lexer.class.__flexr_rules.sort_by(&:index).each do |rule|
+          next unless rule_active?(rule)
+
+          matches = rule.patterns.each_with_index.filter_map do |pattern, pattern_index|
+            condition = rule.pattern_conditions.fetch(pattern_index)
+            next if condition.bol_only && !@lexer.beginning_of_line?
+
+            regexp = pattern.is_a?(::Regexp) ? pattern : ::Regexp.new(::Regexp.escape(pattern.to_s))
+            streamed_match(regexp, buffer, position, reference: reference_pattern?(regexp))
+              &.then { |match| [match, condition] }
+          rescue ArgumentError, RegexpError
+            nil
+          end
+          match, condition = matches.select { |candidate| candidate[0].begin(0).zero? }
+            .max_by { |candidate| candidate[0][0].bytesize }
+          next unless match
+
+          end_position = position + match[0].bytesize
+          next unless @lexer.utf8_boundary?(end_position)
+          next if condition.end_anchor && !end_anchor_match?(buffer, end_position)
+          trailing = trailing_length(rule, buffer, end_position)
+          next if rule.trailing && trailing.nil?
+
+          ensure_token_size!(end_position, position)
+          return reusable_match(rule, position, end_position, end_position + (trailing || 0))
+        end
+        nil
+      end
+
+      def trailing_length(rule, buffer, position)
+        return 0 unless rule.trailing
+
+        regexp = rule.trailing
+        match = streamed_match(regexp, buffer, position, reference: reference_pattern?(regexp))
+        return nil unless match&.begin(0)&.zero?
+
+        match[0].bytesize
+      rescue ArgumentError
+        nil
+      end
+
+      def streamed_match(pattern, buffer, position, reference: false)
+        pattern = regexp_pattern(pattern)
+        minimum = minimum_match_bytes(pattern)
+        loop do
+          subject, tail = stream_subject(buffer, position)
+          match = if reference
+            Unicode::ReferenceRegexp.match(
+              pattern, subject, encoding: @lexer.class.__flexr_config.encoding,
+              options: pattern.options, unicode: @lexer.class.__flexr_config.options[:unicode] == true
+            )
+          else
+            pattern.match(subject, 0)
+          end
+          return match if match && match[0].bytesize < subject.bytesize
+          return match if match && %i[eof invalid].include?(tail)
+          if !match && subject.bytesize >= minimum && tail != :incomplete
+            first_byte = subject.getbyte(0)
+            return nil unless first_byte && possible_first_byte?(pattern, first_byte)
+            return nil if %i[eof invalid].include?(tail)
+          end
+          return match unless can_refill_match?(buffer, position, subject.bytesize, tail)
+        rescue ArgumentError, RegexpError
+          return nil
+        end
+      end
+
+      def stream_subject(buffer, position)
+        return [buffer.byteslice(position...buffer.bytesize).to_s.b, buffer.eof_loaded? ? :eof : :end] unless @lexer.utf8_input?
+
+        cursor = position
+        while cursor < buffer.bytesize
+          status, length = utf8_status(buffer, cursor)
+          break unless status == :complete
+
+          cursor += length
+        end
+        tail = if cursor < buffer.bytesize
+          utf8_status(buffer, cursor).first
+        elsif buffer.eof_loaded?
+          :eof
+        else
+          :end
+        end
+        subject = buffer.byteslice(position, cursor - position).to_s.dup.force_encoding(Encoding::UTF_8)
+        [subject, tail]
+      end
+
+      def utf8_status(buffer, position)
+        return [:eof, 0] if position >= buffer.bytesize
+
+        first = buffer.source.getbyte(position)
+        return [:complete, 1] if first <= 0x7f
+        return [:invalid, 1] unless first.between?(0xc2, 0xf4)
+
+        length = if first <= 0xdf
+          2
+        elsif first <= 0xef
+          3
+        else
+          4
+        end
+        return [:incomplete, length] unless buffer.ensure_available?(position + length)
+        return [:invalid, length] unless buffer.valid_utf8_at?(position)
+
+        [:complete, length]
+      end
+
+      def can_refill_match?(buffer, position, subject_size, tail)
+        target = if tail == :incomplete
+          position + subject_size + 1
+        else
+          buffer.bytesize + 1
+        end
+        buffer.ensure_available?(target)
+      end
+
+      def minimum_match_bytes(pattern)
+        return 1 if posix_pattern?(pattern)
+
+        ast = Regexp::Parser.new(pattern.source, options: pattern.options,
+                                 encoding: pattern.encoding, unicode: true).parse
+        minimum_ast_bytes(ast, ignorecase: pattern.options.anybits?(::Regexp::IGNORECASE))
+      rescue CompileError, RegexpError
+        1
+      end
+
+      def regexp_pattern(pattern)
+        return pattern if pattern.is_a?(::Regexp)
+
+        ::Regexp.new(::Regexp.escape(pattern.to_s))
+      end
+
+      def minimum_ast_bytes(node, ignorecase: false)
+        case node
+        when Regexp::AST::Empty, Regexp::AST::Anchor then 0
+        when Regexp::AST::ByteRange then 1
+        when Regexp::AST::CodepointRange then utf8_length(node.lo)
+        when Regexp::AST::CharClass
+          ranges = node.ranges.flat_map do |range|
+            range.first == Regexp::AST::Property ? property_ranges(range, ignorecase: ignorecase) : [range]
+          end
+          ranges = complement_codepoint_ranges(ranges) if node.negated
+          ranges.map { |lo, _hi| utf8_length(lo) }.min || 1
+        when Regexp::AST::Seq then node.children.sum { |child| minimum_ast_bytes(child, ignorecase: ignorecase) }
+        when Regexp::AST::Alt then node.children.map { |child| minimum_ast_bytes(child, ignorecase: ignorecase) }.min || 0
+        else minimum_unknown_bytes(node)
+        end
+      end
+
+      def minimum_unknown_bytes(node)
+        return 0 if node.is_a?(Regexp::AST::Star)
+
+        1
+      end
+
+      def possible_first_byte?(pattern, byte)
+        return true if posix_pattern?(pattern)
+
+        ast = Regexp::Parser.new(pattern.source, options: pattern.options,
+                                 encoding: pattern.encoding, unicode: true).parse
+        binary = !@lexer.utf8_input?
+        first_byte_ranges(ast, pattern.options, binary: binary).any? { |lo, hi| byte.between?(lo, hi) }
+      rescue CompileError, RegexpError
+        true
+      end
+
+      def first_byte_ranges(node, options, binary: false)
+        case node
+        when Regexp::AST::Empty, Regexp::AST::Anchor then []
+        when Regexp::AST::ByteRange then [[node.lo, node.hi]]
+        when Regexp::AST::CodepointRange
+          ranges = options.anybits?(::Regexp::IGNORECASE) ? Unicode::CaseFold.ranges(node.lo, node.hi) : [[node.lo, node.hi]]
+          return ranges if binary
+
+          ranges.flat_map { |lo, hi| Unicode::Utf8Splitter.split(lo, hi).map(&:first).map(&:first) }
+            .map { |value| [value, value] }
+        when Regexp::AST::CharClass
+          ranges = node.ranges.flat_map do |range|
+            range.first == Regexp::AST::Property ? property_ranges(range, ignorecase: options.anybits?(::Regexp::IGNORECASE)) : [range]
+          end
+          ranges = complement_codepoint_ranges(ranges) if node.negated
+          return ranges if binary
+
+          ranges.flat_map { |lo, hi| Unicode::Utf8Splitter.split(lo, hi).map(&:first).map(&:first) }
+            .map { |value| [value, value] }
+        when Regexp::AST::Seq
+          ranges = []
+          node.children.each do |child|
+            ranges.concat(first_byte_ranges(child, options, binary: binary))
+            break unless nullable?(child)
+          end
+          ranges
+        when Regexp::AST::Alt
+          node.children.flat_map { |child| first_byte_ranges(child, options, binary: binary) }
+        when Regexp::AST::Star
+          first_byte_ranges(node.child, options, binary: binary)
+        else
+          first_byte_fallback
+        end
+      end
+
+      def first_byte_fallback
+        []
+      end
+
+      def nullable?(node)
+        case node
+        when Regexp::AST::Empty, Regexp::AST::Anchor, Regexp::AST::Star then true
+        when Regexp::AST::Seq then node.children.all? { |child| nullable?(child) }
+        when Regexp::AST::Alt then node.children.any? { |child| nullable?(child) }
+        else false
+        end
+      end
+
+      def utf8_length(codepoint)
+        return 1 if codepoint <= 0x7f
+        return 2 if codepoint <= 0x7ff
+        return 3 if codepoint <= 0xffff
+
+        4
+      end
+
+      def complement_codepoint_ranges(ranges)
+        result = []
+        cursor = 0
+        ranges.sort_by(&:first).each do |lo, hi|
+          result << [cursor, lo - 1] if cursor < lo
+          cursor = [cursor, hi + 1].max
+        end
+        result << [cursor, 0x10ffff] if cursor <= 0x10ffff
+        result
+      end
+
+      def property_ranges(range, ignorecase:)
+        ranges = Unicode::Property.ranges(range[2])
+        ranges = Unicode::CaseFold.merge(ranges.flat_map { |lo, hi| Unicode::CaseFold.ranges(lo, hi) }) if ignorecase
+        range[1] ? complement_codepoint_ranges(ranges) : ranges
+      end
+
+      def posix_pattern?(pattern)
+        pattern.is_a?(::Regexp) && pattern.source.match?(/\[:(?:\^)?[a-z]+:\]/)
+      end
+
+      def consider_acceptances(machine, state, cursor, position, buffer, best)
+        machine.dfa.accepts[state].each do |acceptance|
+          candidate = @lexer.class.__flexr_rules.fetch(acceptance.rule_index)
+          next if acceptance.bol_only && !@lexer.beginning_of_line?
+          next unless @lexer.utf8_boundary?(cursor)
+          next unless !acceptance.end_anchor || end_anchor_match?(buffer, cursor)
+
+          trailing_size = trailing_length(candidate, buffer, cursor)
+          next if candidate.trailing && trailing_size.nil?
+
+          total_end = cursor + (trailing_size || 0)
+          next if best && total_end < best.total_end_pos
+          next if best && total_end == best.total_end_pos && acceptance.rule_index > best.rule.index
+
+          ensure_token_size!(cursor, position)
+          best = @match ||= Match.new
+          best.rule = candidate
+          best.start_pos = position
+          best.end_pos = cursor
+          best.total_end_pos = total_end
+        end
+        best
+      end
+
+      def acceleration_enabled?
+        @lexer.class.__flexr_config.options.fetch(:accel, :auto) != :none
+      end
+
+      def reusable_match(rule, start_pos, end_pos, total_end_pos)
+        @match ||= Match.new
+        @match.rule = rule
+        @match.start_pos = start_pos
+        @match.end_pos = end_pos
+        @match.total_end_pos = total_end_pos
+        @match
+      end
+
+      def accelerate(region, buffer, position)
+        mode = @lexer.class.__flexr_config.options.fetch(:accel, :auto)
+        binary = buffer.source.b
+        match_end = if %i[strscan auto].include?(mode) && defined?(StringScanner)
+          scanner = StringScanner.new(binary)
+          scanner.pos = position
+          length = scanner.skip(region.regexp)
+          length && scanner.pos
+        else
+          region.regexp.match(binary, position)&.then { |match| match.begin(0) == position ? match.end(0) : nil }
+        end
+        return match_end if match_end && match_end < buffer.bytesize
+        return match_end if match_end && buffer.eof_loaded?
+        return unless buffer.ensure_available?(buffer.bytesize + 1)
+
+        accelerate(region, buffer, position)
+      rescue ArgumentError
+        nil
+      end
+
+      def transition(dfa, state, byte)
+        if @lexer.class.__flexr_config.backend == :direct &&
+            @lexer.class.respond_to?(:__flexr_generated_direct_transition)
+          return @lexer.class.__flexr_generated_direct_transition(@lexer.state, state, byte)
+        end
+        return dfa.transition_direct(state, byte) if @lexer.class.__flexr_config.backend == :direct
+
+        dfa.transition(state, byte)
+      end
+
+      def acceleration_regions_for(machine)
+        @acceleration_regions ||= {}
+        return @acceleration_regions[machine.dfa] if @acceleration_regions.key?(machine.dfa)
+
+        @acceleration_regions[machine.dfa] = Automaton::Accel.extract(machine.dfa).filter_map do |region|
+          next if region.bytes.any? { |byte| byte >= 128 }
+
+          accepting = machine.dfa.accepts[region.state]
+          next if accepting.any? do |acceptance|
+            rule = @lexer.class.__flexr_rules.fetch(acceptance.rule_index)
+            acceptance.bol_only || acceptance.end_anchor || rule.trailing
+          end
+
+          [region.state, region]
+        end.to_h
+      end
+
+      def end_anchor_match?(buffer, position)
+        buffer.eof?(position) || buffer.getbyte(position) == 0x0a
+      end
+
+      def ensure_token_size!(end_position, position)
+        text_start = @lexer.more_text_start || position
+        @lexer.defer_token_size_check!(end_position - text_start)
+      end
+
+      def rule_active?(rule)
+        return true if rule.states.include?(:initial) && @lexer.state == :initial
+
+        state = @lexer.class.__flexr_config.states.fetch(@lexer.state)
+        return true if state.inclusive && rule.states.include?(:initial)
+
+        rule.states.include?(@lexer.state)
+      end
+
+    end
+  end
+end

data/lib/flexr/runtime/location.rb

@@ -0,0 +1,26 @@
+# frozen_string_literal: true
+
+module Flexr
+  module Runtime
+    Location = Struct.new(
+      :filename, :byte_begin, :byte_end, :line_begin, :column_begin, :line_end, :column_end,
+      keyword_init: true
+    ) do
+      def initialize(**values)
+        @column_values = values.delete(:column_values)
+        eager_columns = values.delete(:eager_columns)
+        super(**values) # rubocop:disable Style/SuperArguments
+        self.column_begin = @column_values.first if eager_columns && @column_values
+        self.column_end = @column_values.last if eager_columns && @column_values
+      end
+
+      def column_begin
+        self[:column_begin] ||= @column_values&.first
+      end
+
+      def column_end
+        self[:column_end] ||= @column_values&.last
+      end
+    end
+  end
+end

data/lib/flexr/runtime/token.rb

@@ -0,0 +1,7 @@
+# frozen_string_literal: true
+
+module Flexr
+  module Runtime
+    Token = Struct.new(:type, :value, :location, keyword_init: true)
+  end
+end

data/lib/flexr/source/passthrough.rb

@@ -0,0 +1,31 @@
+# frozen_string_literal: true
+
+module Flexr
+  module Source
+    module Passthrough
+      module_function
+
+      def remove_spans(source, spans, insertion:, payload:)
+        result = source.dup
+        top_level = spans.reject do |span|
+          spans.any? do |outer|
+            next false if outer.equal?(span)
+
+            outer.first <= span.first && span.last <= outer.last &&
+              (outer.first < span.first || span.last < outer.last)
+          end
+        end
+        top_level.sort_by(&:first).reverse_each do |start_offset, end_offset|
+          result.slice!(start_offset...end_offset)
+        end
+        result.insert(insertion, payload)
+        result
+      end
+
+      def indentation(source, offset)
+        line_start = source.rindex("\n", offset - 1)
+        source[(line_start ? line_start + 1 : 0)...offset].to_s[/\A[ \t]*/].to_s
+      end
+    end
+  end
+end