regular_expression 0.1.0

data/lib/regular_expression/cfg.rb

@@ -0,0 +1,154 @@
+# frozen_string_literal: true
+
+module RegularExpression
+  # The CFG is a directed graph of extended basic blocks of bytecode
+  # instructions. This module has objects to represent the EBB, a graph object
+  # which contains a set of EBB, and a builder that creates a CFG from a
+  # compiled bytecode object.
+  module CFG
+    def self.build(compiled)
+      # Each label in the compiled bytecode starts a block, as does the first
+      # instruction
+      all_blocks = { start: 0 }.merge(compiled.labels)
+      all_block_addresses = all_blocks.values
+
+      # We're going to create a potentially larger map of labels, and we'll be
+      # maintaining a reverse map as well.
+      all_labels = compiled.labels.dup
+      all_labels_reverse = all_labels.invert
+
+      # These are the blocks we're finding - indexed by their start address.
+      blocks = {}
+
+      # Go through each block.
+      all_blocks.each do |name, start_n|
+        # We're going to collect up the instructions in the block, and the
+        # labels it exits to.
+        block_insns = []
+        block_exits = Set.new
+
+        insn_n = start_n
+
+        loop do
+          # Does another instruction jump here? If so it's the end of the EBB,
+          # as EBBs have only one entry point.
+          if insn_n != start_n && all_block_addresses.include?(insn_n)
+            # As the EBB ends here - we should jump to the next EBB.
+            target = all_labels_reverse[insn_n]
+            unless target
+              target = :"extra#{insn_n}"
+              all_labels[target] = insn_n
+              all_labels_reverse[insn_n] = target
+            end
+            block_insns.push(Bytecode::Insns::Jump.new(target))
+            block_exits.add(target)
+            break
+          end
+
+          # Examine each instruction.
+          insn = compiled.insns[insn_n]
+          block_insns.push(insn)
+
+          # Remember which blocks exit to this target.
+          case insn
+          when Bytecode::Insns::PushIndex, Bytecode::Insns::PopIndex
+            insn_n += 1
+          when Bytecode::Insns::GuardBegin, Bytecode::Insns::GuardEnd
+            block_exits.add(insn.guarded)
+            insn_n += 1
+          when Bytecode::Insns::JumpAny, Bytecode::Insns::JumpValuesInvert,
+                Bytecode::Insns::JumpRange, Bytecode::Insns::JumpRangeInvert,
+                Bytecode::Insns::JumpValue
+            block_exits.add(insn.target)
+            insn_n += 1
+          when Bytecode::Insns::Jump
+            block_exits.add(insn.target)
+            break
+          when Bytecode::Insns::Match, Bytecode::Insns::Fail
+            break
+          else
+            raise
+          end
+        end
+
+        blocks[start_n] = ExtendedBasicBlock.new(name, block_insns, block_exits.to_a)
+      end
+
+      # Create a map of jump target labels to the blocks that contain them.
+      exit_map = {}
+      blocks.each_value do |block|
+        block.exits.each do |exit|
+          exit_map[exit] ||= blocks[all_labels[exit]]
+        end
+      end
+
+      Graph.new(blocks.values, exit_map)
+    end
+
+    def self.to_dot(cfg)
+      graph = Graphviz::Graph.new
+      cfg.to_dot(graph)
+
+      Graphviz.output(graph, path: "build/cfg.svg", format: "svg")
+      graph.to_dot
+    end
+
+    # An Extended Basic Block is a linear sequence of instructions with one
+    # entry point and zero or more exit points.
+    class ExtendedBasicBlock
+      attr_reader :name, :insns, :exits
+
+      def initialize(name, insns, exits)
+        @name = name
+        @insns = insns
+        @exits = exits
+      end
+
+      def dump(exit_map, io: $stdout)
+        io.puts("#{name}:")
+        insns.each { |insn| io.puts("  #{insn}") }
+        exits.each { |exit| io.puts("    #{exit} -> #{exit_map[exit].name}") }
+      end
+    end
+
+    # A graph is a set of EBBs.
+    class Graph
+      attr_reader :blocks, :exit_map
+
+      def initialize(blocks, exit_map)
+        @blocks = blocks
+        @exit_map = exit_map
+      end
+
+      def start
+        blocks.first
+      end
+
+      def dump
+        output = StringIO.new
+        blocks.each { |block| block.dump(exit_map, io: output) }
+        output.string
+      end
+
+      def to_dot(graph)
+        nodes = {}
+
+        blocks.each do |block|
+          label = []
+
+          label.push("#{block.name}:")
+          block.insns.each { |insn| label.push("  #{insn}") }
+
+          nodes[block] = graph.add_node(block.object_id, label: label.join($/), labeljust: "l", shape: "box")
+        end
+
+        blocks.each do |block|
+          successors = block.exits.map { |exit| nodes[exit_map[exit]] }.uniq
+          successors.each do |successor|
+            nodes[block].connect(successor)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/regular_expression/compiler/ruby.rb

@@ -0,0 +1,104 @@
+# frozen_string_literal: true
+
+module RegularExpression
+  module Compiler
+    module Ruby
+      class Compiled
+        attr_reader :source
+
+        def initialize(source)
+          @source = source
+        end
+
+        def to_proc
+          eval(source) # rubocop:disable Security/Eval
+        end
+      end
+
+      # Generate Ruby code for a CFG. This looks just like the intepreter, but
+      # abstracted in time one level!
+      # rubocop:disable Layout/LineLength
+      def self.compile(cfg)
+        ruby_src = []
+        ruby_src.push "-> (string) {"
+        ruby_src.push "  start_n = 0"
+        ruby_src.push "  stack = []"
+        ruby_src.push "  while start_n <= string.size"
+        ruby_src.push "    string_n = start_n"
+        ruby_src.push "    block = #{cfg.start.name.inspect}"
+        ruby_src.push "    loop do"
+        ruby_src.push "      case block"
+
+        cfg.blocks.each do |block|
+          ruby_src.push "      when #{block.name.inspect}"
+
+          block.insns.each do |insn|
+            case insn
+            when Bytecode::Insns::PushIndex
+              ruby_src.push "        stack << string_n"
+            when Bytecode::Insns::PopIndex
+              ruby_src.push "        string_n = stack.pop"
+            when Bytecode::Insns::GuardBegin
+              ruby_src.push "        return false if start_n != 0"
+            when Bytecode::Insns::GuardEnd
+              ruby_src.push "        if string_n == string.size"
+              ruby_src.push "          block = #{cfg.exit_map[insn.guarded].name.inspect}"
+              ruby_src.push "          next"
+              ruby_src.push "        end"
+            when Bytecode::Insns::JumpAny
+              ruby_src.push "        if string_n < string.size"
+              ruby_src.push "          string_n += 1"
+              ruby_src.push "          block = #{cfg.exit_map[insn.target].name.inspect}"
+              ruby_src.push "          next"
+              ruby_src.push "        end"
+            when Bytecode::Insns::JumpValue
+              ruby_src.push "        if string_n < string.size && string[string_n] == #{insn.char.inspect}"
+              ruby_src.push "          string_n += 1"
+              ruby_src.push "          block = #{cfg.exit_map[insn.target].name.inspect}"
+              ruby_src.push "          next"
+              ruby_src.push "        end"
+            when Bytecode::Insns::JumpValuesInvert
+              ruby_src.push "        if string_n < string.size && !#{insn.chars.inspect}.include?(string[string_n])"
+              ruby_src.push "          string_n += 1"
+              ruby_src.push "          block = #{cfg.exit_map[insn.target].name.inspect}"
+              ruby_src.push "          next"
+              ruby_src.push "        end"
+            when Bytecode::Insns::JumpRange
+              ruby_src.push "        if string_n < string.size && string[string_n] >= #{insn.left.inspect} && string[string_n] <= #{insn.right.inspect}"
+              ruby_src.push "          string_n += 1"
+              ruby_src.push "          block = #{cfg.exit_map[insn.target].name.inspect}"
+              ruby_src.push "          next"
+              ruby_src.push "        end"
+            when Bytecode::Insns::JumpRangeInvert
+              ruby_src.push "        if string_n < string.size && (string[string_n] < #{insn.left.inspect} || string[string_n] > #{insn.right.inspect})"
+              ruby_src.push "          string_n += 1"
+              ruby_src.push "          block = #{cfg.exit_map[insn.target].name.inspect}"
+              ruby_src.push "          next"
+              ruby_src.push "        end"
+            when Bytecode::Insns::Jump
+              ruby_src.push "        block = #{cfg.exit_map[insn.target].name.inspect}"
+              ruby_src.push "        next"
+            when Bytecode::Insns::Match
+              ruby_src.push "        return true"
+            when Bytecode::Insns::Fail
+              ruby_src.push "        start_n += 1"
+              ruby_src.push "        break"
+            else
+              raise
+            end
+          end
+        end
+
+        ruby_src.push "      end"
+        ruby_src.push "    end"
+        ruby_src.push "  end"
+        ruby_src.push "  false"
+        ruby_src.push "}"
+        ruby_src.push ""
+
+        Compiled.new(ruby_src.join($/))
+      end
+      # rubocop:enable Layout/LineLength
+    end
+  end
+end

data/lib/regular_expression/compiler/x86.rb

@@ -0,0 +1,281 @@
+# frozen_string_literal: true
+
+module RegularExpression
+  module Compiler
+    module X86
+      class Compiled
+        attr_reader :buffer
+
+        def initialize(buffer)
+          @buffer = buffer
+        end
+
+        def disasm
+          output = StringIO.new
+
+          crabstone = Crabstone::Disassembler.new(Crabstone::ARCH_X86, Crabstone::MODE_64)
+          crabstone.disasm(buffer.memory.to_s(buffer.pos), buffer.memory.to_i).each do |insn|
+            output.printf(
+              "0x%<address>x:\t%<instruction>s\t%<details>s\n",
+              address: insn.address,
+              instruction: insn.mnemonic,
+              details: insn.op_str
+            )
+          end
+
+          output.string
+        end
+
+        def to_proc
+          function = buffer.to_function([Fiddle::TYPE_VOIDP, Fiddle::TYPE_SIZE_T], Fiddle::TYPE_SIZE_T)
+
+          lambda do |string|
+            value = function.call(string, string.length)
+            value if value != string.length + 1
+          end
+        end
+      end
+
+      # Generate native code for a CFG. This looks just like the Ruby generator
+      # but abstracted one level, or just like the interpreter but abstracted
+      # two levels!
+      def self.compile(cfg)
+        fisk = Fisk.new
+        buffer = Fisk::Helpers.jitbuffer(1024)
+
+        fisk.asm(buffer) do
+          # Here we're setting up a couple of local variables that point to
+          # registers so that it's easier to see what's actually going on
+
+          # rax is a scratch register that is used for the return value of the
+          # function
+          return_value = rax
+
+          # rcx is a scratch register that is used to track the index of the
+          # string where we're currently looking
+          string_index = rcx
+
+          # rdx is a scratch register that is used to track the index of the
+          # string where we've started the match
+          match_index = rdx
+
+          # rsp is a reserved register that stores a pointer to the stack
+          stack_pointer = rsp
+
+          # rbp is a reserved register that stores a pointer to the base of the
+          # stack. It is also known as the frame pointer
+          frame_pointer = rbp
+
+          # rsi is a scratch register that stores the second argument to the
+          # function, and in our case stores the length of the string
+          string_length = rsi
+
+          # rdi is a scratch register that stores the first argument to the
+          # function, and in our case stores a pointer to the base of the string
+          string_pointer = rdi
+
+          # r8 is a scratch register that we're using to store the last read
+          # character value from the string
+          character_buffer = r8
+
+          # First we're going to do some initialization of the frame pointer and
+          # stack pointer so we can clear the stack when we're done with this
+          # function
+          push frame_pointer
+          mov frame_pointer, stack_pointer
+
+          # Now we're going to initialize the counter to 0 so that we attempt to
+          # match at each index of the input string
+          xor match_index, match_index
+
+          # This is the start of our loop, where at the beginning of the loop
+          # we check if we have already finished looking at each index (in which
+          # case we'll jump to a failure condition)
+          make_label :start_loop_head
+          cmp match_index, string_length
+          jg label(:exit)
+
+          # Set the string_index value to the match_index value so that we begin
+          # each loop at the current match index
+          mov string_index, match_index
+
+          cfg.blocks.each do |block|
+            # Label the start of each block so that we can jump between them
+            make_label block.name
+
+            block.insns.each do |insn|
+              case insn
+              when Bytecode::Insns::PushIndex
+                push string_index
+              when Bytecode::Insns::PopIndex
+                pop string_index
+              when Bytecode::Insns::GuardBegin
+                cmp string_index, imm8(0)
+                jne label(:exit)
+                jmp label(cfg.exit_map[insn.guarded].name)
+              when Bytecode::Insns::GuardEnd
+                cmp string_index, string_length
+                je label(cfg.exit_map[insn.guarded].name)
+              when Bytecode::Insns::JumpAny
+                no_match_label = :"no_match_#{insn.object_id}"
+
+                # Ensure we have a character we can read
+                cmp string_index, string_length
+                je label(no_match_label)
+
+                # Move the string index forward and jump to the target
+                # instruction
+                inc string_index
+                jmp label(cfg.exit_map[insn.target].name)
+
+                make_label no_match_label
+              when Bytecode::Insns::JumpValue
+                no_match_label = :"no_match_#{insn.object_id}"
+
+                # Ensure we have a character we can read
+                cmp string_index, string_length
+                je label(no_match_label)
+
+                # Read the character into the character buffer
+                mov character_buffer, string_pointer
+                add character_buffer, string_index
+                mov character_buffer, m64(character_buffer)
+
+                # Compare the character buffer to the instruction's character,
+                # continue on to the next instruction if it's not equal
+                cmp character_buffer, imm8(insn.char.ord)
+                jne label(no_match_label)
+
+                # Move the string index forward and jump to the target
+                # instruction
+                inc string_index
+                jmp label(cfg.exit_map[insn.target].name)
+
+                make_label no_match_label
+              when Bytecode::Insns::JumpValuesInvert
+                no_match_label = :"no_match_#{insn.object_id}"
+
+                # Ensure we have a character we can read
+                cmp string_index, string_length
+                je label(no_match_label)
+
+                # Read the character into the character buffer
+                mov character_buffer, string_pointer
+                add character_buffer, string_index
+                mov character_buffer, m64(character_buffer)
+
+                # Compare the character buffer to each of the instruction's
+                # characters, continue on to the next instruction if any of them
+                # are equal
+                insn.chars.each do |value|
+                  cmp character_buffer, imm8(value.ord)
+                  je label(no_match_label)
+                end
+
+                # Move the string index forward and jump to the target
+                # instruction
+                inc string_index
+                jmp label(cfg.exit_map[insn.target].name)
+
+                make_label no_match_label
+              when Bytecode::Insns::JumpRange
+                no_match_label = :"no_match_#{insn.object_id}"
+
+                # Ensure we have a character we can read
+                cmp string_index, string_length
+                je label(no_match_label)
+
+                # Read the character into the character buffer
+                mov character_buffer, string_pointer
+                add character_buffer, string_index
+                mov character_buffer, m64(character_buffer)
+
+                # Compare the character buffer to the left hand side of the
+                # instruction's range, continue on to the next instruction if
+                # it's outside the range
+                cmp character_buffer, imm8(insn.left.ord)
+                jl label(no_match_label)
+
+                # Compare the character buffer to the right hand side of the
+                # instruction's range, continue on to the next instruction if
+                # it's outside the range
+                cmp character_buffer, imm8(insn.right.ord)
+                jg label(no_match_label)
+
+                # Move the string index forward and jump to the target
+                # instruction
+                inc string_index
+                jmp label(cfg.exit_map[insn.target].name)
+
+                make_label no_match_label
+              when Bytecode::Insns::JumpRangeInvert
+                no_match_label = :"no_match_#{insn.object_id}"
+                match_label = :"match_#{insn.object_id}"
+
+                # Ensure we have a character we can read
+                cmp string_index, string_length
+                je label(no_match_label)
+
+                # Read the character into the character buffer
+                mov character_buffer, string_pointer
+                add character_buffer, string_index
+                mov character_buffer, m64(character_buffer)
+
+                # Compare the character buffer to the left hand side of the
+                # instruction's range, jump down to the success case if it's
+                # outside the range
+                cmp character_buffer, imm8(insn.left.ord)
+                jl label(match_label)
+
+                # Compare the character buffer to the right hand side of the
+                # instruction's range, continue on to the next instruction if
+                # it's inside the range
+                cmp character_buffer, imm8(insn.right.ord)
+                jle label(no_match_label)
+
+                # Move the string index forward and jump to the target
+                # instruction
+                make_label match_label
+                inc string_index
+                jmp label(cfg.exit_map[insn.target].name)
+
+                make_label no_match_label
+              when Bytecode::Insns::Jump
+                jmp label(cfg.exit_map[insn.target].name)
+              when Bytecode::Insns::Match
+                # If we reach this instruction, then we've successfully matched
+                # against the input string, so we're going to return the integer
+                # that represents the index at which this match began
+                mov return_value, match_index
+                mov stack_pointer, frame_pointer
+                pop frame_pointer
+                ret
+              when Bytecode::Insns::Fail
+                inc match_index
+                jmp label(:start_loop_head)
+              else
+                raise
+              end
+            end
+          end
+
+          # If we reach this instruction, then we've failed to match at every
+          # possible index in the string, so we're going to return the length
+          # of the string + 1 so that the caller knows that this match failed
+          make_label :exit
+          mov return_value, string_length
+          inc return_value
+
+          # Here we make sure to clean up after ourselves by returning the frame
+          # pointer to its former position
+          mov stack_pointer, frame_pointer
+          pop frame_pointer
+
+          ret
+        end
+
+        Compiled.new(buffer)
+      end
+    end
+  end
+end