regular_expression 0.1.0

data/bin/parse

@@ -0,0 +1,50 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+
+$:.unshift(File.expand_path("../lib", __dir__))
+require "regular_expression"
+require "crabstone"
+require "graphviz"
+
+# Pass the source through the various parsing phases
+source = ARGV.shift
+ast = RegularExpression::Parser.new.parse(source)
+nfa = ast.to_nfa
+bytecode = RegularExpression::Bytecode.compile(nfa)
+
+# Compile the graph into various outputs
+cfg = RegularExpression::CFG.build(bytecode)
+ruby = RegularExpression::Compiler::Ruby.compile(cfg)
+x86 = RegularExpression::Compiler::X86.compile(cfg)
+
+# Make sure we get some nice dot output
+RegularExpression::AST.to_dot(ast)
+RegularExpression::NFA.to_dot(nfa)
+RegularExpression::CFG.to_dot(cfg)
+
+# Dump out the bytecode and cfg to strings
+puts "#{bytecode.dump}\n"
+puts "#{cfg.dump}\n"
+
+check =
+  if ARGV.any?
+    lambda do |compiled|
+      checker = compiled.to_proc
+      ARGV.each { |string| puts "#{string}: #{checker.call(string)}" }
+      puts
+    end
+  else
+    ->(_compiled) {}
+  end
+
+# Test the interpreter against any passed strings
+interpreter = RegularExpression::Interpreter.new(bytecode)
+check.call(interpreter)
+
+# Dump out the compiled ruby source and match against any passed values
+puts "#{ruby.source}\n"
+check.call(ruby)
+
+# Dump out the diassembled x86 source and match against any passed values
+puts "#{x86.disasm}\n"
+check.call(x86)

data/build/.gitignore

@@ -0,0 +1 @@
+*.svg

data/lib/regular_expression.rb

@@ -0,0 +1,19 @@
+# frozen_string_literal: true
+
+require "fisk"
+require "fisk/helpers"
+require "set"
+require "stringio"
+
+require_relative "./regular_expression/ast"
+require_relative "./regular_expression/bytecode"
+require_relative "./regular_expression/cfg"
+require_relative "./regular_expression/interpreter"
+require_relative "./regular_expression/lexer"
+require_relative "./regular_expression/nfa"
+require_relative "./regular_expression/parser"
+require_relative "./regular_expression/pattern"
+require_relative "./regular_expression/version"
+
+require_relative "./regular_expression/compiler/ruby"
+require_relative "./regular_expression/compiler/x86"

data/lib/regular_expression/ast.rb

@@ -0,0 +1,364 @@
+# frozen_string_literal: true
+
+module RegularExpression
+  module AST
+    def self.to_dot(root)
+      graph = Graphviz::Graph.new
+      root.to_dot(graph)
+
+      Graphviz.output(graph, path: "build/ast.svg", format: "svg")
+      graph.to_dot
+    end
+
+    class Root
+      attr_reader :expressions # Array[Expression]
+      attr_reader :at_start # bool
+
+      def initialize(expressions, at_start: false)
+        @expressions = expressions
+        @at_start = at_start
+      end
+
+      def to_dot(graph)
+        label = "Root"
+        label = "#{label} (at start)" if at_start
+
+        node = graph.add_node(object_id, label: label)
+        expressions.each { |expression| expression.to_dot(node) }
+      end
+
+      def to_nfa
+        start = NFA::StartState.new
+        current = start
+
+        if at_start
+          current = NFA::State.new
+          start.add_transition(NFA::Transition::BeginAnchor.new(current))
+        end
+
+        finish = NFA::FinishState.new
+        expressions.each do |expression|
+          expression.to_nfa(current, finish)
+        end
+
+        start
+      end
+    end
+
+    class Expression
+      attr_reader :items # Group | Match | Anchor
+
+      def initialize(items)
+        @items = items
+      end
+
+      def to_dot(parent)
+        node = parent.add_node(object_id, label: "Expression")
+
+        items.each { |item| item.to_dot(node) }
+      end
+
+      def to_nfa(start, finish)
+        inner = Array.new(items.length - 1) { NFA::State.new }
+        states = [start, *inner, finish]
+
+        items.each_with_index do |item, index|
+          item.to_nfa(states[index], states[index + 1])
+        end
+      end
+    end
+
+    class Group
+      attr_reader :expressions # Array[Expression]
+      attr_reader :quantifier # Quantifier
+
+      def initialize(expressions, quantifier: Quantifier::Once.new)
+        @expressions = expressions
+        @quantifier = quantifier
+      end
+
+      def to_dot(parent)
+        node = parent.add_node(object_id, label: "Group")
+
+        expressions.each { |expression| expression.to_dot(node) }
+        quantifier.to_dot(node)
+      end
+
+      def to_nfa(start, finish)
+        quantifier.quantify(start, finish) do |qstart, qfinish|
+          expressions.each { |expression| expression.to_nfa(qstart, qfinish) }
+        end
+      end
+    end
+
+    class Match
+      attr_reader :item # CharacterGroup | CharacterClass | Character | Period
+      attr_reader :quantifier # Quantifier
+
+      def initialize(item, quantifier: Quantifier::Once.new)
+        @item = item
+        @quantifier = quantifier
+      end
+
+      def to_dot(parent)
+        node = parent.add_node(object_id, label: "Match")
+
+        item.to_dot(node)
+        quantifier.to_dot(node)
+      end
+
+      def to_nfa(start, finish)
+        quantifier.quantify(start, finish) do |qstart, qfinish|
+          item.to_nfa(qstart, qfinish)
+        end
+      end
+    end
+
+    class CharacterGroup
+      attr_reader :items # Array[CharacterRange | Character]
+      attr_reader :invert # bool
+
+      def initialize(items, invert: false)
+        @items = items
+        @invert = invert
+      end
+
+      def to_dot(parent)
+        label = "CharacterGroup"
+        label = "#{label} (invert)" if invert
+
+        node = parent.add_node(object_id, label: label)
+        items.each { |item| item.to_dot(node) }
+      end
+
+      def to_nfa(start, finish)
+        if invert
+          transition = NFA::Transition::Invert.new(finish, items.flat_map(&:to_nfa_values).sort)
+          start.add_transition(transition)
+        else
+          items.each do |item|
+            item.to_nfa(start, finish)
+          end
+        end
+      end
+    end
+
+    class CharacterClass
+      attr_reader :value # "\w" | "\W" | "\d" | "\D"
+
+      def initialize(value)
+        @value = value
+      end
+
+      def to_dot(parent)
+        parent.add_node(object_id, label: value, shape: "box")
+      end
+
+      def to_nfa(start, finish)
+        case value
+        when "\\w"
+          start.add_transition(NFA::Transition::Range.new(finish, "a", "z"))
+          start.add_transition(NFA::Transition::Range.new(finish, "A", "Z"))
+          start.add_transition(NFA::Transition::Range.new(finish, "0", "9"))
+          start.add_transition(NFA::Transition::Value.new(finish, "_"))
+        when "\\W"
+          start.add_transition(NFA::Transition::Invert.new(finish, [*("a".."z"), *("A".."Z"), *("0".."9"), "_"]))
+        when "\\d"
+          start.add_transition(NFA::Transition::Range.new(finish, "0", "9"))
+        when "\\D"
+          start.add_transition(NFA::Transition::Range.new(finish, "0", "9", invert: true))
+        else
+          raise
+        end
+      end
+    end
+
+    class Character
+      attr_reader :value # String
+
+      def initialize(value)
+        @value = value
+      end
+
+      def to_dot(parent)
+        parent.add_node(object_id, label: value, shape: "box")
+      end
+
+      def to_nfa_values
+        [value]
+      end
+
+      def to_nfa(start, finish)
+        start.add_transition(NFA::Transition::Value.new(finish, value))
+      end
+    end
+
+    class Period
+      def to_dot(parent)
+        parent.add_node(object_id, label: ".", shape: "box")
+      end
+
+      def to_nfa(start, finish)
+        transition = NFA::Transition::Any.new(finish)
+        start.add_transition(transition)
+      end
+    end
+
+    class CharacterRange
+      attr_reader :left, :right # String
+
+      def initialize(left, right)
+        @left = left
+        @right = right
+      end
+
+      def to_dot(parent)
+        parent.add_node(object_id, label: "#{left}-#{right}", shape: "box")
+      end
+
+      def to_nfa_values
+        (left..right).to_a
+      end
+
+      def to_nfa(start, finish)
+        transition = NFA::Transition::Range.new(finish, left, right)
+        start.add_transition(transition)
+      end
+    end
+
+    class Anchor
+      attr_reader :value # "\A" | "\z" | "$"
+
+      def initialize(value)
+        @value = value
+      end
+
+      def to_dot(parent)
+        parent.add_node(object_id, label: value, shape: "box")
+      end
+
+      def to_nfa(start, finish)
+        transition =
+          case value
+          when "\\A"
+            NFA::Transition::BeginAnchor.new(finish)
+          when "\\z", "$"
+            NFA::Transition::EndAnchor.new(finish)
+          end
+
+        start.add_transition(transition)
+      end
+    end
+
+    module Quantifier
+      class Once
+        def to_dot(parent); end
+
+        def quantify(start, finish)
+          yield start, finish
+        end
+      end
+
+      class ZeroOrMore
+        def to_dot(parent)
+          parent.add_node(object_id, label: "*", shape: "box")
+        end
+
+        def quantify(start, finish)
+          yield start, start
+          start.add_transition(NFA::Transition::Epsilon.new(finish))
+        end
+      end
+
+      class OneOrMore
+        def to_dot(parent)
+          parent.add_node(object_id, label: "+", shape: "box")
+        end
+
+        def quantify(start, finish)
+          yield start, finish
+          finish.add_transition(NFA::Transition::Epsilon.new(start))
+        end
+      end
+
+      class Optional
+        def to_dot(parent)
+          parent.add_node(object_id, label: "?", shape: "box")
+        end
+
+        def quantify(start, finish)
+          yield start, finish
+          start.add_transition(NFA::Transition::Epsilon.new(finish))
+        end
+      end
+
+      class Exact
+        attr_reader :value # Integer
+
+        def initialize(value)
+          @value = value
+        end
+
+        def to_dot(parent)
+          parent.add_node(object_id, label: "{#{value}}", shape: "box")
+        end
+
+        def quantify(start, finish)
+          states = [start, *(value - 1).times.map { NFA::State.new }, finish]
+
+          value.times do |index|
+            yield states[index], states[index + 1]
+          end
+        end
+      end
+
+      class AtLeast
+        attr_reader :value # Integer
+
+        def initialize(value)
+          @value = value
+        end
+
+        def to_dot(parent)
+          parent.add_node(object_id, label: "{#{value},}", shape: "box")
+        end
+
+        def quantify(start, finish)
+          states = [start, *(value - 1).times.map { NFA::State.new }, finish]
+
+          value.times do |index|
+            yield states[index], states[index + 1]
+          end
+
+          states[-1].add_transition(NFA::Transition::Epsilon.new(states[-2]))
+        end
+      end
+
+      class Range
+        attr_reader :lower, :upper # Integer
+
+        def initialize(lower, upper)
+          @lower = lower
+          @upper = upper
+        end
+
+        def to_dot(parent)
+          parent.add_node(object_id, label: "{#{lower},#{upper}}", shape: "box")
+        end
+
+        def quantify(start, finish)
+          states = [start, *(upper - 1).times.map { NFA::State.new }, finish]
+
+          upper.times do |index|
+            yield states[index], states[index + 1]
+          end
+
+          (upper - lower).times do |index|
+            transition = NFA::Transition::Epsilon.new(states[-1])
+            states[lower + index].add_transition(transition)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/regular_expression/bytecode.rb

@@ -0,0 +1,189 @@
+# frozen_string_literal: true
+
+module RegularExpression
+  # The bytecode module defines instructions, and has a compiled object for
+  # storing a stream of them, and a builder object for creating the compiled
+  # object.
+  module Bytecode
+    # Never recurse a graph in a compiler! We don't know how deep it is and
+    # don't want to limit how large a program we can accept due to arbitrary
+    # stack space. Always use a worklist.
+    def self.compile(nfa)
+      builder = Builder.new
+      label = ->(state, index = 0) { :"state_#{state.object_id}_#{index}" }
+
+      visited = Set.new
+      worklist = [[nfa, [Insns::Jump.new(:fail)]]]
+
+      # For each state in the NFA.
+      until worklist.empty?
+        state, fallback = worklist.pop
+        next if visited.include?(state)
+
+        # Label the start of the state.
+        builder.mark_label(label[state])
+        visited.add(state)
+
+        if state.is_a?(NFA::FinishState)
+          builder.push(Insns::Match.new)
+          next
+        end
+
+        # Other states have transitions out of them. Go through each
+        # transition.
+        state.transitions.each_with_index do |transition, index|
+          builder.mark_label(label[state, index])
+
+          if state.transitions.length > 1 && index != state.transitions.length - 1
+            builder.push(Insns::PushIndex.new)
+          end
+
+          case transition
+          when NFA::Transition::BeginAnchor
+            builder.push(Insns::GuardBegin.new(label[transition.state]))
+          when NFA::Transition::EndAnchor
+            builder.push(Insns::GuardEnd.new(label[transition.state]))
+          when NFA::Transition::Any
+            builder.push(Insns::JumpAny.new(label[transition.state]))
+          when NFA::Transition::Value
+            builder.push(Insns::JumpValue.new(transition.value, label[transition.state]))
+          when NFA::Transition::Invert
+            builder.push(Insns::JumpValuesInvert.new(transition.values, label[transition.state]))
+          when NFA::Transition::Range
+            if transition.invert
+              builder.push(Insns::JumpRangeInvert.new(transition.left, transition.right, label[transition.state]))
+            else
+              builder.push(Insns::JumpRange.new(transition.left, transition.right, label[transition.state]))
+            end
+          when NFA::Transition::Epsilon
+            builder.push(Insns::Jump.new(label[transition.state]))
+          else
+            raise
+          end
+
+          next_fallback =
+            if state.transitions.length > 1 && index != state.transitions.length - 1
+              [Insns::PopIndex.new, Insns::Jump.new(label[state, index + 1])]
+            else
+              fallback
+            end
+
+          worklist.push([transition.state, next_fallback])
+        end
+
+        # If we don't have one of the transitions that always executes, then we
+        # need to add the fallback to the output for this state.
+        if state.transitions.none? { |t| t.is_a?(NFA::Transition::BeginAnchor) || t.is_a?(NFA::Transition::Epsilon) }
+          builder.push(*fallback)
+        end
+      end
+
+      # We always have a failure case - it's just the failure instruction.
+      builder.mark_label(:fail)
+      builder.push(Insns::Fail.new)
+      builder.build
+    end
+
+    module Insns
+      # Push the current string index onto the stack. This is necessary to
+      # support backtracking so that we can pop it off later when we want to go
+      # backward.
+      PushIndex = Class.new
+
+      # Pop the string index off the stack. This is necessary so that we can
+      # support backtracking.
+      PopIndex = Class.new
+
+      # If we're at the beginning of the string, then jump to the then
+      # instruction. Otherwise fail the entire match.
+      GuardBegin = Struct.new(:guarded)
+
+      # If we're at the end of the string, then jump to the then instruction.
+      # Otherwise fail the match at the current index.
+      GuardEnd = Struct.new(:guarded)
+
+      # If it's possible to read a character off the input, then do so and jump
+      # to the target instruction.
+      JumpAny = Struct.new(:target)
+
+      # If it's possible to read a character off the input and that character
+      # matches the char value, then do so and jump to the target instruction.
+      JumpValue = Struct.new(:char, :target)
+
+      # If it's possible to read a character off the input and that character is
+      # not contained within the list of values, then do so and jump to the
+      # target instruction.
+      JumpValuesInvert = Struct.new(:chars, :target)
+
+      # If it's possible to read a character off the input and that character is
+      # within the range of possible values, then do so and jump to the target
+      # instruction.
+      JumpRange = Struct.new(:left, :right, :target)
+
+      # If it's possible to read a character off the input and that character is
+      # not within the range of possible values, then do so and jump to the
+      # target instruction.
+      JumpRangeInvert = Struct.new(:left, :right, :target)
+
+      # Jump directly to the target instruction.
+      Jump = Struct.new(:target)
+
+      # Successfully match the string and stop executing instructions.
+      Match = Class.new
+
+      # Fail to match the string at the current index. Increment the starting
+      # index and try again if possible.
+      Fail = Class.new
+    end
+
+    class Builder
+      attr_reader :insns # Array[Insns]
+      attr_reader :labels # Hash[Symbol, Integer]
+
+      def initialize
+        @insns = []
+        @labels = {}
+      end
+
+      def mark_label(label)
+        labels[label] = insns.size
+      end
+
+      def push(*new_insns)
+        insns.push(*new_insns)
+      end
+
+      def build
+        Compiled.new(insns, labels)
+      end
+    end
+
+    class Compiled
+      attr_reader :insns, :labels
+
+      def initialize(insns, labels)
+        @insns = insns
+        @labels = labels
+      end
+
+      def dump
+        output = StringIO.new
+
+        # Labels store name -> address, but if we want to print the label name
+        # at its address, we need to store the address to the name as well.
+        reverse_labels = {}
+        labels.each do |label, n|
+          reverse_labels[n] = label
+        end
+
+        insns.each_with_index do |insn, n|
+          label = reverse_labels[n]
+          output.puts("#{label}:") if label
+          output.puts("  #{insn}")
+        end
+
+        output.string
+      end
+    end
+  end
+end