flexr 1.0.0 → 1.1.0

data/lib/flexr/generated.rb

@@ -2,8 +2,20 @@
 
 module Flexr
   module Generated
+    ARTIFACT_SCHEMA_VERSION = 1
+    RUNTIME_ABI_VERSION = 1
+
     module_function
 
+    def artifact_metadata
+      {
+        schema_version: ARTIFACT_SCHEMA_VERSION,
+        compiler_version: VERSION,
+        runtime_abi_version: RUNTIME_ABI_VERSION,
+        unicode_version: Unicode::VERSION
+      }
+    end
+
     def install!(klass, payload)
       rules = payload.fetch(:rules)
       config = payload
@@ -34,20 +46,21 @@ module Flexr
     end
 
     def install_compiled!(klass, payload)
+      validate_artifact!(payload[:artifact])
       install!(klass, payload)
       machines = payload.fetch(:compiled).fetch(:machines).transform_values do |machine|
         dfa_data = machine.fetch(:dfa)
         packed = decode_packed(dfa_data[:packed])
         dfa = Automaton::DFA.new(
-          transitions: dfa_data[:transitions] || inflate_packed(packed, dfa_data.fetch(:state_count),
-                                                                 dfa_data.fetch(:class_count)),
+          transitions: dfa_data[:transitions],
           accepts: dfa_data.fetch(:accepts),
           ec: dfa_data.fetch(:ec),
           class_count: dfa_data.fetch(:class_count),
           start: dfa_data.fetch(:start),
           rule_ids: dfa_data.fetch(:rule_ids),
           packed: packed,
-          direct: dfa_data[:direct]
+          direct: dfa_data[:direct],
+          state_count: dfa_data.fetch(:state_count)
         )
         Automaton::Machine.new(dfa: dfa, state_name: machine.fetch(:state_name).to_sym)
       end
@@ -65,6 +78,35 @@ module Flexr
       klass
     end
 
+    def validate_artifact!(artifact)
+      return incompatible_artifact!("artifact metadata is missing") unless artifact.is_a?(Hash)
+
+      expected = {
+        schema_version: ARTIFACT_SCHEMA_VERSION,
+        runtime_abi_version: RUNTIME_ABI_VERSION,
+        unicode_version: Unicode::VERSION
+      }
+      mismatch = expected.find { |name, value| artifact[name] != value }
+      compiler_version = artifact[:compiler_version]
+      return if mismatch.nil? && compiler_version.is_a?(String) && !compiler_version.empty?
+
+      detail = if mismatch
+        name, value = mismatch
+        "#{name}=#{artifact[name].inspect} (expected #{value.inspect})"
+      else
+        "compiler_version=#{compiler_version.inspect}"
+      end
+      incompatible_artifact!(detail)
+    end
+
+    def incompatible_artifact!(detail)
+      diagnostic = Diagnostics.error(
+        "FLEXR-E020", "generated artifact is incompatible with this runtime",
+        help: "regenerate the lexer with the installed flexr version", note: detail
+      )
+      raise CompileError.new(diagnostic.message, diagnostic: diagnostic)
+    end
+
     def decode_packed(packed)
       return packed unless packed.is_a?(Hash) && packed[:encoding]&.to_sym == :base64
 
@@ -84,42 +126,26 @@ module Flexr
       values.map { |value| value == nil_value ? nil : value }
     end
 
-    def inflate_packed(packed, state_count, class_count)
-      return nil unless packed
-
-      Array.new(state_count) do |state|
-        Array.new(class_count) do |class_id|
-          cursor = state
-          loop do
-            index = packed.fetch(:base).fetch(cursor) + class_id
-            break packed.fetch(:next).fetch(index) if packed.fetch(:check)[index] == cursor
-
-            fallback = packed[:fallback]&.fetch(cursor)
-            break packed.fetch(:default).fetch(cursor) unless fallback
-
-            cursor = fallback
-          end
-        end
-      end
-    end
   end
 
   module DSL
     def __flexr_add_generated_rule(definition)
-      action = definition.fetch(:action)
-      @__flexr_rules << IR::Rule.new(
-        index: definition.fetch(:index), patterns: Array(definition.fetch(:patterns)),
-        trailing: normalize_trailing(definition[:trailing]), action: action,
-        states: Array(definition.fetch(:states)).map(&:to_sym),
-        bol_only: definition.fetch(:bol_only, false), end_anchor: definition[:end_anchor],
-        location: definition[:span] || definition[:location],
-        pattern_conditions: Array(definition[:pattern_conditions]).map do |condition|
-          next unless condition
-
-          Automaton::Acceptance.new(rule_index: condition[0], pattern_index: condition[1],
-                                    bol_only: condition[2], end_anchor: condition[3])
-        end
-      )
+      __flexr_mutate! do
+        action = definition.fetch(:action)
+        @__flexr_rules << IR::Rule.new(
+          index: definition.fetch(:index), patterns: Array(definition.fetch(:patterns)),
+          trailing: normalize_trailing(definition[:trailing]), action: action,
+          states: Array(definition.fetch(:states)).map(&:to_sym),
+          bol_only: definition.fetch(:bol_only, false), end_anchor: definition[:end_anchor],
+          location: definition[:span] || definition[:location],
+          pattern_conditions: Array(definition[:pattern_conditions]).map do |condition|
+            next unless condition
+
+            Automaton::Acceptance.new(rule_index: condition[0], pattern_index: condition[1],
+                                      bol_only: condition[2], end_anchor: condition[3])
+          end
+        )
+      end
     end
   end
 end

data/lib/flexr/generator.rb

@@ -1,5 +1,11 @@
 # frozen_string_literal: true
 
+require "digest"
+require_relative "runtime" unless defined?(Flexr::Lexer)
+require_relative "artifact_writer"
+require_relative "codegen"
+require_relative "source"
+
 module Flexr
   class Generator
     attr_reader :diagnostics
@@ -8,15 +14,18 @@ module Flexr
     INLINE_EMIT_ARGUMENTS = /\A(?::[A-Za-z_]\w*|true|false|nil|-?\d+(?:\.\d+)?|"(?:\\.|[^"])*"|'(?:\\.|[^'])*'|text(?:\.(?:to_f|to_i|bytesize|byteslice\([^()\n]+\)))?|lineno|line)(?:\s*,\s*(?::[A-Za-z_]\w*|true|false|nil|-?\d+(?:\.\d+)?|"(?:\\.|[^"])*"|'(?:\\.|[^'])*'|text(?:\.(?:to_f|to_i|bytesize|byteslice\([^()\n]+\)))?|lineno|line))*\z/
 
     RUNTIME_SOURCES = %w[
-      version.rb errors.rb diagnostics.rb ir.rb
-      regexp/ast.rb regexp/parser.rb regexp/normalizer.rb regexp/unsupported.rb regexp/char_class.rb
-      unicode/utf8_splitter.rb unicode/data/properties.rb unicode/data/case_folding.rb unicode/property.rb unicode/reference_regexp.rb unicode/case_fold.rb
-      automaton/byte_class_set.rb automaton/nfa.rb automaton/dfa.rb automaton/compiler.rb
-      automaton/analysis.rb automaton/minimizer.rb automaton/accel.rb
+      version.rb errors.rb diagnostics.rb action_resolver.rb configuration.rb ir.rb
+      unicode/version.rb automaton/types.rb automaton/dfa.rb automaton/analysis.rb automaton/accel.rb
       runtime/location.rb runtime/token.rb runtime/buffer.rb runtime/errors.rb
       runtime/interpreter.rb runtime/core.rb dsl.rb lexer.rb generated.rb
     ].freeze
 
+    MATCHER_RUNTIME_SOURCES = %w[
+      regexp/ast.rb regexp/parser.rb regexp/unsupported.rb
+      unicode/utf8_splitter.rb unicode/data/properties.rb unicode/data/case_folding.rb
+      unicode/property.rb unicode/reference_regexp.rb unicode/case_fold.rb
+    ].freeze
+
     def initialize(path, output: nil, eval_mode: false, options: {})
       @path = path
       @output = output
@@ -26,6 +35,7 @@ module Flexr
     end
 
     def generate
+      ArtifactWriter.ensure_distinct!(@path, @output) if @output
       # Keep generated source stable when a checkout presents the spec with
       # Windows line endings. All offsets consumed by the parser and
       # passthrough writer are relative to this normalized source.
@@ -36,7 +46,7 @@ module Flexr
         parsed = Source::PrismReader.new(source, path: @path).read
         generate_static(parsed)
       end
-      File.binwrite(@output, result) if @output
+      ArtifactWriter.write!(@output, result, source_path: @path) if @output
       result
     end
 
@@ -51,24 +61,26 @@ module Flexr
       install = "Flexr::Generated.install_compiled!(self, #{payload})\n"
       install = "#{install}#{Codegen::Table.new(compiled).source(indent: indent)}"
       install = "#{install}#{generated_action_source(parsed, indent)}"
-      install = "#{install}#{Codegen::Direct.new(compiled).source(indent: indent)}#{indent}" if
-        effective_backend(parsed) == :direct
-      result = Source::Passthrough.remove_spans(parsed.source, parsed.dsl_spans, insertion: parsed.first_dsl_offset, payload: install)
-      result = result.gsub(/^[ \t]+(?=\n)/, "")
-      result = result.gsub(/^\s*require ["']flexr["']\s*\n/, "") if standalone?(parsed)
-      digest = Digest::SHA256.hexdigest(payload)
+      edits = parsed.dsl_edits.dup
+      edits.concat(parsed.flexr_require_spans.map { |span| [*span, ""] }) if standalone?(parsed)
+      result = Source::Passthrough.rewrite(
+        parsed.source, edits, insertion: parsed.first_dsl_offset, payload: install
+      )
+      digest = Digest::SHA256.hexdigest(result)
       header = [
         "# Generated by flexr. DO NOT EDIT.",
         "# source: #{parsed.path}",
         "# spec-digest: sha256:#{digest}",
+        "# artifact-schema: #{Generated::ARTIFACT_SCHEMA_VERSION}",
+        "# runtime-abi: #{Generated::RUNTIME_ABI_VERSION}",
         "# unicode: #{Unicode::VERSION}",
         "# backend: #{effective_backend(parsed)}",
         "# compiled: true",
         "# eval: #{@eval_mode}",
         "# standalone: #{standalone?(parsed)}"
       ].join("\n")
-      prefix = standalone?(parsed) ? "require \"json\"\n#{embedded_runtime}\n" : ""
-      result.sub(/\A(# frozen_string_literal: true\n)/) { "#{::Regexp.last_match(1)}#{header}\n#{prefix}" }
+      prefix = standalone?(parsed) ? "require \"json\"\nrequire \"monitor\"\n#{embedded_runtime(parsed)}\n" : ""
+      Source::Passthrough.insert_after_preamble(result, "#{header}\n#{prefix}")
     end
 
     def generated_payload(parsed, compiled)
@@ -80,7 +92,7 @@ module Flexr
           "action: #{action_expression(rule.action)}, states: #{rule.states.inspect}, " \
           "bol_only: #{ruby_literal(rule.bol_only)}, end_anchor: #{ruby_literal(rule.end_anchor)} }"
       end
-      "{ rules: [#{definitions.join(', ')}], " \
+      "{ artifact: #{ruby_literal(artifact_metadata)}, rules: [#{definitions.join(', ')}], " \
         "backend: #{ruby_literal(effective_backend(parsed))}, " \
         "token_kind: #{ruby_literal(@options.fetch(:token_kind, parsed.config[:token_kind]))}, " \
         "encoding: #{encoding_expression(parsed.config[:encoding])}, " \
@@ -93,6 +105,10 @@ module Flexr
                                          backend: effective_backend(parsed))} }"
     end
 
+    def artifact_metadata
+      Generated.artifact_metadata
+    end
+
     def compile_parsed(parsed)
       states = parsed.states.each_with_index.to_h do |(name, value), index|
         [name.to_sym, IR::State.new(name: name.to_sym, inclusive: value[:inclusive], id: index)]
@@ -111,7 +127,6 @@ module Flexr
         eof_rules: parsed.config[:eof_rules] || {}, verbatim: parsed.source
       )
       compiled = Automaton::Compiler.new(spec).compile
-      validate_firstmatch_equivalence!(spec, compiled) if spec.backend == :firstmatch
       @resolved_backend = resolve_backend(spec.backend, compiled)
       parsed.rules.each do |rule|
         compiled_rule = spec.rules.fetch(rule.index)
@@ -128,8 +143,7 @@ module Flexr
       return requested unless requested == :auto
       return :table unless compiled
 
-      cells = compiled.stats.values.map { |stats| stats[:states] * stats[:classes] }.max.to_i
-      cells > DSL::AUTO_DIRECT_CELL_THRESHOLD ? :direct : :table
+      Automaton::BackendCostModel.choose(compiled)
     end
 
     def normalize_trailing(value)
@@ -144,66 +158,6 @@ module Flexr
       end
     end
 
-    def validate_firstmatch_equivalence!(spec, compiled)
-      return if spec.rules.any? do |rule|
-        rule.trailing || rule.patterns.any? { |pattern| reference_pattern?(pattern, unicode: spec.options[:unicode] == true) }
-      end
-
-      machine = compiled.machines.fetch(:initial).dfa
-      random = Random.new(17)
-      inputs = ["", "a", "aa", "aaa", "ba", "aab"]
-      inputs.concat(Array.new(9_994) do
-        Array.new(random.rand(10)) { random.rand(32..126) }.pack("C*")
-      end)
-      inputs.each do |input|
-        table = table_match(machine, input)
-        firstmatch = firstmatch_match(spec.rules, input)
-        next if table == firstmatch
-
-        raise CompileError, "firstmatch differs from table for #{input.inspect}: #{firstmatch.inspect} vs #{table.inspect}"
-      end
-    end
-
-    def reference_pattern?(pattern, unicode: false)
-      return false unless pattern.is_a?(::Regexp)
-      return true if pattern.source.match?(/\\[pP]\{/) || pattern.source.match?(/\[:(?:\^)?[a-z]+:\]/)
-
-      unicode && pattern.encoding != Encoding::BINARY && pattern.source.match?(/\\[dDwWsS]/)
-    end
-
-    def table_match(dfa, input)
-      state = dfa.start
-      best = nil
-      input.each_byte.with_index do |byte, index|
-        state = dfa.transition(state, byte)
-        break unless state
-
-        acceptance = dfa.accepts[state].select do |candidate|
-          !candidate.end_anchor || index + 1 == input.bytesize || input.getbyte(index + 1) == 0x0a
-        end.min_by(&:rule_index)
-        next unless acceptance
-
-        candidate = [acceptance.rule_index, index + 1]
-        best = candidate if best.nil? || candidate[1] > best[1] ||
-          (candidate[1] == best[1] && candidate[0] < best[0])
-      end
-      best
-    end
-
-    def firstmatch_match(rules, input)
-      rules.sort_by(&:index).each do |rule|
-        lengths = rule.patterns.filter_map do |pattern|
-          regexp = pattern.is_a?(::Regexp) ? pattern : ::Regexp.new(::Regexp.escape(pattern.to_s))
-          match = regexp.match(input, 0)
-          match&.begin(0)&.zero? ? match[0].bytesize : nil
-        rescue ArgumentError, RegexpError
-          nil
-        end
-        return [rule.index, lengths.max] unless lengths.empty?
-      end
-      nil
-    end
-
     def compiled_expression(compiled, compression: :none, backend: :table)
       machines = compiled.machines.map do |name, machine|
         dfa = machine.dfa
@@ -216,7 +170,9 @@ module Flexr
           rule_ids: dfa.rule_ids
         }
         pack_tables = %i[rows full].include?(compression) || @options.fetch(:table_format, :literal).to_sym == :packed
-        if pack_tables
+        if backend == :direct
+          data[:direct] = Codegen::Direct.new(compiled).generate(state: name)
+        elsif pack_tables
           packed = Codegen::TablePacker.pack(dfa.transitions, compression: compression)
           data[:packed] = if @options.fetch(:table_format, :literal).to_sym == :packed
             Codegen::TablePacker.encode(packed)
@@ -226,7 +182,6 @@ module Flexr
         else
           data[:transitions] = dfa.transitions
         end
-        data[:direct] = Codegen::Direct.new(compiled).generate(state: name) if backend == :direct
         "#{ruby_literal(name)} => { state_name: #{ruby_literal(machine.state_name)}, dfa: #{ruby_literal(data)} }"
       end
       # W016 is derived from wall-clock compilation time and must not make
@@ -253,12 +208,18 @@ module Flexr
       @options.fetch(:table_compression, parsed.config[:options]&.fetch(:table_compression, :rows) || :rows).to_sym
     end
 
-    def embedded_runtime
-      RUNTIME_SOURCES.map do |relative|
+    def embedded_runtime(parsed)
+      sources = RUNTIME_SOURCES.dup
+      sources.insert(6, *MATCHER_RUNTIME_SOURCES) if matcher_runtime?(parsed)
+      sources.map do |relative|
         File.binread(File.expand_path(relative, __dir__))
       end.join("\n")
     end
 
+    def matcher_runtime?(parsed)
+      effective_backend(parsed) == :firstmatch || parsed.rules.any?(&:trailing)
+    end
+
     def encoding_expression(encoding)
       encoding == Encoding::BINARY ? "Encoding::BINARY" : "Encoding::UTF_8"
     end
@@ -279,6 +240,10 @@ module Flexr
       lines = ["#{indent}def __flexr_generated_execute(rule)", "#{indent}  case rule.index"]
       parsed.rules.each do |rule|
         lines << "#{indent}  when #{rule.index}"
+        if rule.bind_action
+          lines << "#{indent}    instance_exec(&self.class.__flexr_generated_actions.fetch(rule.index))"
+          next
+        end
         body = inline_action_body(rule)
         if body.nil?
           lines << "#{indent}    instance_exec(&rule.action)"

data/lib/flexr/importer.rb

@@ -1,5 +1,7 @@
 # frozen_string_literal: true
 
+require_relative "runtime" unless defined?(Flexr::Lexer)
+
 module Flexr
   class Importer
     Result = Struct.new(:source, :warnings, :complete?, keyword_init: true)
@@ -387,46 +389,8 @@ module Flexr
 
     def rexical_dfa_counterexample(regexps, automata)
       first, second = automata
-      queue = [[first.start, second.start, false, +"".b]]
-      visited = { [first.start, second.start, false] => true }
-      bytes = rexical_byte_representatives(first, second)
-
-      until queue.empty?
-        first_state, second_state, first_seen, input = queue.shift
-        bytes.each do |byte|
-          next_first = first_state && first.transition(first_state, byte)
-          next_second = second.transition(second_state, byte)
-          next unless next_second
-
-          next_input = input + byte.chr(Encoding::BINARY)
-          next_first_seen = first_seen || rexical_accepting?(first, next_first)
-          if next_first_seen && rexical_accepting?(second, next_second) &&
-              !rexical_accepting?(first, next_first)
-            lengths = rexical_match_lengths(regexps, next_input)
-            return lengths if lengths
-          end
-
-          key = [next_first, next_second, next_first_seen]
-          next if visited[key]
-
-          visited[key] = true
-          queue << [next_first, next_second, next_first_seen, next_input]
-        end
-      end
-      nil
-    end
-
-    def rexical_byte_representatives(first, second)
-      representatives = {}
-      256.times do |byte|
-        key = [first.ec[byte], second.ec[byte]]
-        representatives[key] ||= byte
-      end
-      representatives.values.sort_by { |byte| [byte.between?(32, 126) ? 0 : 1, byte] }
-    end
-
-    def rexical_accepting?(dfa, state)
-      state && !dfa.accepts.fetch(state).empty?
+      input = Automaton::Analysis.firstmatch_counterexample(first, second)
+      input && rexical_match_lengths(regexps, input)
     end
 
     def rexical_match_lengths(regexps, input)

data/lib/flexr/options.rb

@@ -1,5 +1,7 @@
 # frozen_string_literal: true
 
+require_relative "runtime" unless defined?(Flexr::Lexer)
+
 module Flexr
   Options = Struct.new(
     :backend, :token_kind, :accel, :standalone, :eval_mode, :table_compression,
@@ -20,9 +22,9 @@ module Flexr
     end
 
     def validate!
-      validate_value!(backend, %i[table direct firstmatch auto], :backend)
-      validate_value!(token_kind, %i[array struct yield], :token_kind)
-      validate_value!(accel, %i[auto strscan regexp none], :accel)
+      validate_value!(backend, Configuration::BACKENDS, :backend)
+      validate_value!(token_kind, Configuration::TOKEN_KINDS, :token_kind)
+      validate_value!(accel, Configuration::ACCELERATORS, :accel)
       validate_value!(table_compression, %i[none rows full], :table_compression)
       validate_value!(table_format, %i[literal packed], :table_format)
       validate_value!(warn_level, %i[all default none], :warn_level)

data/lib/flexr/rake_task.rb

@@ -1,5 +1,9 @@
 # frozen_string_literal: true
 
+require_relative "runtime" unless defined?(Flexr::Lexer)
+require_relative "artifact_writer"
+require_relative "generator"
+
 module Flexr
   class RakeTask
     attr_accessor :spec, :output, :warn_as_error
@@ -17,7 +21,8 @@ module Flexr
       require "rake"
       raise ArgumentError, "spec is required" unless @spec
 
-      output = @output || @spec.sub(/\.flexr\.rb\z/, ".rb")
+      output = @output || ArtifactWriter.default_generated_path(@spec)
+      ArtifactWriter.ensure_distinct!(@spec, output)
       Rake::FileTask.define_task(output => @spec) do
         Generator.new(@spec, output: output, options: { warn_as_error: @warn_as_error }).generate
       end

data/lib/flexr/regexp/ast.rb

@@ -4,11 +4,13 @@ module Flexr
   module Regexp
     module AST
       Empty = Struct.new(:loc, keyword_init: true)
+      Fail = Struct.new(:loc, keyword_init: true)
       ByteRange = Struct.new(:lo, :hi, :loc, keyword_init: true)
       CodepointRange = Struct.new(:lo, :hi, :loc, keyword_init: true)
       Seq = Struct.new(:children, :loc, keyword_init: true)
       Alt = Struct.new(:children, :loc, keyword_init: true)
       Star = Struct.new(:child, :loc, keyword_init: true)
+      Repeat = Struct.new(:child, :minimum, :maximum, :loc, keyword_init: true)
       Anchor = Struct.new(:kind, :loc, keyword_init: true)
       TrailMark = Struct.new(:rule_id, :loc, keyword_init: true)
       CharClass = Struct.new(:ranges, :negated, :loc, keyword_init: true)
@@ -18,18 +20,19 @@ module Flexr
         def to_s
           name = self.class.name.split("::").last
           case name
-          when "Empty" then "Empty"
+          when "Empty", "Fail" then name
           when "ByteRange" then "ByteRange(#{lo}..#{hi})"
           when "CodepointRange" then "CodepointRange(#{lo}..#{hi})"
           when "Anchor" then "Anchor(#{kind})"
           when "Star" then "Star(#{child})"
+          when "Repeat" then "Repeat(#{child}, #{minimum}, #{maximum || '∞'})"
           when "Seq", "Alt" then "#{name}(#{children.join(', ')})"
           else super
           end
         end
       end
 
-      [Empty, ByteRange, CodepointRange, Seq, Alt, Star, Anchor, TrailMark, CharClass].each do |node_class|
+      [Empty, Fail, ByteRange, CodepointRange, Seq, Alt, Star, Repeat, Anchor, TrailMark, CharClass].each do |node_class|
         node_class.include(Formatting)
       end
 

data/lib/flexr/regexp/normalizer.rb

@@ -18,7 +18,7 @@ module Flexr
 
       def normalize_node(node)
         case node
-        when AST::Empty, AST::ByteRange, AST::CodepointRange, AST::Anchor
+        when AST::Empty, AST::Fail, AST::ByteRange, AST::CodepointRange, AST::Anchor
           normalize_leaf(node)
         when AST::CharClass
           char_class(node)
@@ -27,7 +27,10 @@ module Flexr
         when AST::Alt
           alternatives(node.children.map { |child| normalize_node(child) })
         when AST::Star
-          AST::Star.new(child: normalize_node(node.child), loc: node.loc)
+          child = normalize_node(node.child)
+          child.is_a?(AST::Fail) ? AST::Empty.new(loc: node.loc) : AST::Star.new(child: child, loc: node.loc)
+        when AST::Repeat
+          repeat(node)
         else
           raise CompileError, "unknown regexp AST node: #{node.class}"
         end
@@ -36,7 +39,7 @@ module Flexr
       def normalize_leaf(node)
         case node
         when AST::CodepointRange
-          byte_sequences_for_ranges(casefold_ranges(node.lo, node.hi))
+          byte_sequences_for_ranges(casefold_ranges(node.lo, node.hi), loc: node.loc)
         else
           node
         end
@@ -45,44 +48,76 @@ module Flexr
       def char_class(node)
         ranges = node.ranges.flat_map do |range|
           if range.first == AST::Property
-            Unicode::Property.ranges(range.last, negate: range[1])
+            property_ranges = Unicode::Property.ranges(range.last)
+            property_ranges = property_ranges.flat_map { |lo, hi| casefold_ranges(lo, hi) }
+            range[1] ? complement(property_ranges) : property_ranges
           else
             casefold_ranges(*range)
           end
         end
         ranges = complement(ranges) if node.negated
-        byte_sequences_for_ranges(ranges)
+        byte_sequences_for_ranges(ranges, loc: node.loc)
       end
 
-      def byte_sequences_for_ranges(ranges)
+      def byte_sequences_for_ranges(ranges, loc:)
         if @byte_mode
           byte_ranges = ranges.filter_map do |lo, hi|
             next if lo > 255
 
             [lo, [hi, 255].min]
           end
-          return AST::Empty.new(loc: nil) if byte_ranges.empty?
+          return AST::Fail.new(loc: loc) if byte_ranges.empty?
 
-          return alternatives(byte_ranges.map { |lo, hi| AST::ByteRange.new(lo: lo, hi: hi, loc: nil) })
+          return alternatives(byte_ranges.map { |lo, hi| AST::ByteRange.new(lo: lo, hi: hi, loc: loc) })
         end
 
         sequences = ranges.flat_map { |lo, hi| Unicode::Utf8Splitter.split(lo, hi) }
+        @trie_cache = {}
         trie(sequences, 0)
       end
 
       def trie(sequences, index)
-        return AST::Empty.new(loc: nil) if sequences.empty?
-        return AST::Empty.new(loc: nil) if sequences.first.length == index
-
-        groups = sequences.group_by { |sequence| sequence[index] }
-        alternatives(groups.map do |(lo, hi), group|
-          child = trie(group, index + 1)
-          AST::Seq.new(children: [AST::ByteRange.new(lo: lo, hi: hi, loc: nil), child], loc: nil)
+        return AST::Fail.new(loc: nil) if sequences.empty?
+
+        finished, continuing = sequences.partition { |sequence| sequence.length == index }
+        nodes = []
+        nodes << AST::Empty.new(loc: nil) unless finished.empty?
+        return nodes.first if continuing.empty?
+
+        key = [index, continuing]
+        return alternatives(nodes + [@trie_cache[key]]) if @trie_cache.key?(key)
+
+        endpoints = continuing.flat_map do |sequence|
+          lo, hi = sequence.fetch(index)
+          [lo, hi + 1]
+        end.uniq.sort
+        partitions = endpoints.each_cons(2).filter_map do |lo, after_hi|
+          covered = continuing.select do |sequence|
+            range = sequence.fetch(index)
+            range.first <= lo && range.last >= after_hi - 1
+          end
+          [lo, after_hi - 1, covered] unless covered.empty?
+        end
+        branches = partitions.map { |lo, hi, covered| [lo, hi, trie(covered, index + 1)] }
+        merged = branches.each_with_object([]) do |(lo, hi, child), result|
+          if !result.empty? && result.last[1] + 1 == lo && result.last[2] == child
+            result.last[1] = hi
+          else
+            result << [lo, hi, child]
+          end
+        end
+        tree = alternatives(merged.map do |lo, hi, child|
+          sequence([AST::ByteRange.new(lo: lo, hi: hi, loc: nil), child])
         end)
+        @trie_cache[key] = tree
+        alternatives(nodes + [tree])
       end
 
       def sequence(children)
         flattened = children.flat_map { |child| child.is_a?(AST::Seq) ? child.children : [child] }
+        return AST::Fail.new(loc: nil) if flattened.any?(AST::Fail)
+
+        flattened.reject! { |child| child.is_a?(AST::Empty) }
         return AST::Empty.new(loc: nil) if flattened.empty?
         return flattened.first if flattened.length == 1
 
@@ -91,12 +126,23 @@ module Flexr
 
       def alternatives(children)
         flattened = children.flat_map { |child| child.is_a?(AST::Alt) ? child.children : [child] }
-        return AST::Empty.new(loc: nil) if flattened.empty?
+        flattened.reject! { |child| child.is_a?(AST::Fail) }
+        flattened.uniq!
+        return AST::Fail.new(loc: nil) if flattened.empty?
         return flattened.first if flattened.length == 1
 
         AST::Alt.new(children: flattened, loc: nil)
       end
 
+      def repeat(node)
+        child = normalize_node(node.child)
+        return AST::Empty.new(loc: node.loc) if node.maximum&.zero? || child.is_a?(AST::Empty)
+        return node.minimum.zero? ? AST::Empty.new(loc: node.loc) : AST::Fail.new(loc: node.loc) if
+          child.is_a?(AST::Fail)
+
+        AST::Repeat.new(child: child, minimum: node.minimum, maximum: node.maximum, loc: node.loc)
+      end
+
       def complement(ranges)
         result = []
         cursor = 0