hoozuki 0.1.0 → 1.0.0

data/lib/hoozuki/parser.y

@@ -0,0 +1,128 @@
+# frozen_string_literal: true
+
+class Hoozuki::Parser
+rule
+  target: choice
+
+  choice:
+      concatenation
+    | choice PIPE concatenation {
+            children = val[0].is_a?(Hoozuki::Node::Choice) ? val[0].children.dup : [val[0]]
+            children << val[2]
+            result = Hoozuki::Node::Choice.new(children)
+          }
+
+  concatenation:
+      repetition
+    | EPSILON { result = Hoozuki::Node::Epsilon.new }
+    | concatenation repetition {
+        if val[0].is_a?(Hoozuki::Node::Epsilon)
+          result = val[1]
+        else
+          children = val[0].is_a?(Hoozuki::Node::Concatenation) ? val[0].children.dup : [val[0]]
+          children << val[1]
+          result = Hoozuki::Node::Concatenation.new(children)
+        end
+      }
+
+  repetition:
+      group
+    | group STAR { result = Hoozuki::Node::Repetition.new(val[0], :zero_or_more) }
+    | group PLUS { result = Hoozuki::Node::Repetition.new(val[0], :one_or_more) }
+    | group QUESTION { result = Hoozuki::Node::Repetition.new(val[0], :optional) }
+
+  group:
+      LPAREN choice RPAREN { result = val[1] }
+    | literal
+
+  literal:
+      CHAR { result = Hoozuki::Node::Literal.new(val[0]) }
+
+end
+
+---- header
+require_relative 'node'
+
+---- inner
+  def initialize
+    @yydebug = true if ENV['DEBUG']
+  end
+
+  def parse(pattern)
+    @pattern = pattern
+    @offset = 0
+    @tokens = []
+    tokenize
+    do_parse
+  end
+
+  private
+
+  ESCAPABLE_CHARS = ['(', ')', '|', '*', '+', '?', '\\'].freeze
+  SPECIAL_TOKENS = {
+    '(' => :LPAREN,
+    ')' => :RPAREN,
+    '|' => :PIPE,
+    '*' => :STAR,
+    '+' => :PLUS,
+    '?' => :QUESTION
+  }.freeze
+
+  def tokenize
+    while @offset < @pattern.length
+      char = @pattern[@offset]
+
+      if char == '\\'
+        handle_escape_sequence
+      elsif SPECIAL_TOKENS.key?(char)
+        handle_special_char(char)
+      else
+        add_token(:CHAR, char)
+      end
+    end
+
+    @tokens << [false, false]
+  end
+
+  def handle_escape_sequence
+    @offset += 1
+    raise 'Unexpected end of pattern' if @offset >= @pattern.length
+
+    escaped = @pattern[@offset]
+    raise "Invalid escape sequence: \\#{escaped}" unless ESCAPABLE_CHARS.include?(escaped)
+
+    add_token(:CHAR, escaped)
+  end
+
+  def handle_special_char(char)
+    token_type = SPECIAL_TOKENS[char]
+    add_token(token_type, char)
+
+    insert_epsilon_after_lparen if char == '(' && next_char == '|'
+    insert_epsilon_after_pipe if char == '|' && should_insert_epsilon_after_pipe?
+  end
+
+  def should_insert_epsilon_after_pipe?
+    next_char.nil? || [')', '|'].include?(next_char)
+  end
+
+  def insert_epsilon_after_lparen
+    @tokens << [:EPSILON, nil]
+  end
+
+  def insert_epsilon_after_pipe
+    @tokens << [:EPSILON, nil]
+  end
+
+  def add_token(type, value)
+    @tokens << [type, value]
+    @offset += 1
+  end
+
+  def next_char
+    @offset < @pattern.length ? @pattern[@offset] : nil
+  end
+
+  def next_token
+    @tokens.shift
+  end

data/lib/hoozuki/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
-class Hoozuki
-  VERSION = '0.1.0'
+module Hoozuki
+  VERSION = '1.0.0'
 end

data/lib/hoozuki/vm/compiler.rb

@@ -0,0 +1,129 @@
+# frozen_string_literal: true
+
+module Hoozuki
+  module VM
+    class Compiler
+      attr_reader :instructions
+
+      def initialize
+        @pc = 0
+        @instructions = []
+      end
+
+      def compile(ast)
+        compile_node(ast)
+        emit_match
+      end
+
+      private
+
+      def compile_node(ast)
+        case ast
+        when Hoozuki::Node::Literal
+          compile_literal(ast)
+        when Hoozuki::Node::Epsilon
+          # Do nothing for epsilon
+        when Node::Repetition
+          compile_repetition(ast)
+        when Node::Choice
+          compile_choice(ast)
+        when Node::Concatenation
+          compile_concatenation(ast)
+        end
+      end
+
+      def compile_literal(node)
+        emit(Hoozuki::Instruction::Char.new(node.value))
+      end
+
+      def compile_repetition(node)
+        if node.zero_or_more?
+          compile_zero_or_more(node.child)
+        elsif node.one_or_more?
+          compile_one_or_more(node.child)
+        elsif node.optional?
+          compile_optional(node.child)
+        end
+      end
+
+      def compile_zero_or_more(child)
+        split = @pc
+        emit(Hoozuki::Instruction::Split.new(@pc + 1, 0))
+        compile_node(child)
+        emit(Hoozuki::Instruction::Jmp.new(split))
+        patch(split, Hoozuki::Instruction::Split.new(split + 1, @pc))
+      end
+
+      def compile_one_or_more(child)
+        start = @pc
+        compile_node(child)
+        emit(Hoozuki::Instruction::Split.new(start, @pc + 1))
+      end
+
+      def compile_optional(child)
+        split = @pc
+        emit(Hoozuki::Instruction::Split.new(0, 0))
+        start = @pc
+        compile_node(child)
+        last = @pc
+        patch(split, Hoozuki::Instruction::Split.new(start, last))
+      end
+
+      def compile_choice(node)
+        if node.children.length == 2
+          compile_binary_choice(node.children[0], node.children[1])
+        else
+          first = node.children[0]
+          rest = Node::Choice.new(node.children[1..])
+          compile_binary_choice(first, rest)
+        end
+      end
+
+      def compile_binary_choice(left, right)
+        split = @pc
+        @pc += 1
+        @instructions << Hoozuki::Instruction::Split.new(@pc, 0)
+        compile_node(left)
+        jump = @pc
+        emit(Hoozuki::Instruction::Jmp.new(0))
+        validate_split_instruction(split)
+        @instructions[split].right = @pc
+        compile_node(right)
+        validate_jmp_instruction(jump)
+        @instructions[jump].target = @pc
+      end
+
+      def compile_concatenation(node)
+        node.children.each do |child|
+          compile_node(child)
+        end
+      end
+
+      def emit_match
+        @pc += 1
+        @instructions << Instruction::Match.new
+      end
+
+      def validate_split_instruction(pc)
+        return if @instructions[pc].is_a?(Hoozuki::Instruction::Split)
+
+        raise "Instruction at pc #{pc} is not a Split"
+      end
+
+      def validate_jmp_instruction(pc)
+        return if @instructions[pc].is_a?(Hoozuki::Instruction::Jmp)
+
+        raise "Instruction at pc #{pc} is not a Jmp"
+      end
+
+      def emit(instruction)
+        @instructions << instruction
+        @pc += 1
+      end
+
+      def patch(pc, instruction)
+        @instructions[pc] = instruction
+      end
+    end
+  end
+end

data/lib/hoozuki/vm/evaluator.rb

@@ -0,0 +1,39 @@
+# frozen_string_literal: true
+
+module Hoozuki
+  module VM
+    class Evaluator
+      class << self
+        def evaluate(instructions, input, input_pos = 0, pc = 0)
+          new._evaluate(instructions, input, input_pos, pc)
+        end
+      end
+
+      def _evaluate(instructions, input, input_pos, pc)
+        loop do
+          return false if pc >= instructions.size
+
+          inst = instructions[pc]
+          case inst
+          when Hoozuki::Instruction::Char
+            return false if input_pos >= input.size || input[input_pos] != inst.char
+
+            input_pos += 1
+            pc += 1
+          when Hoozuki::Instruction::Jmp
+            pc = inst.target
+          when Hoozuki::Instruction::Split
+            return true if _evaluate(instructions, input, input_pos, inst.left)
+
+            pc = inst.right
+
+          when Hoozuki::Instruction::Match
+            return input_pos == input.length
+          else
+            raise "Unknown instruction: #{inst.class}"
+          end
+        end
+      end
+    end
+  end
+end

data/lib/hoozuki/vm.rb

@@ -0,0 +1,4 @@
+# frozen_string_literal: true
+
+require_relative 'vm/compiler'
+require_relative 'vm/evaluator'

data/lib/hoozuki.rb

@@ -1,35 +1,42 @@
 # frozen_string_literal: true
 
 require_relative 'hoozuki/automaton'
+require_relative 'hoozuki/instruction'
 require_relative 'hoozuki/node'
 require_relative 'hoozuki/parser'
 require_relative 'hoozuki/version'
+require_relative 'hoozuki/vm'
 
-class Hoozuki
-  def initialize(input, method: :dfa)
-    @input = input
-    @method = method
+module Hoozuki
+  module_function
 
-    ast = Hoozuki::Parser.new(input).parse
-    case method
+  def compile(input, engine: :dfa)
+    ast = Parser.new.parse(input)
+    case engine
     when :dfa
-      nfa = Automaton::NFA.new_from_node(ast, Automaton::StateID.new(0))
-      @dfa = Automaton::DFA.from_nfa(nfa, use_cache?(input))
+      nfa = Automaton::NFA.from_node(ast, Automaton::StateID.new(0))
+      Automaton::DFA.from_nfa(nfa, use_cache?(input))
+    when :vm
+      compiler = VM::Compiler.new
+      compiler.compile(ast)
+      compiler.instructions
+    else
+      raise ArgumentError, "Unknown engine: #{engine}"
     end
   end
 
-  def match?(input)
-    case @method
+  def match?(pattern, input, engine: :dfa)
+    compiled = compile(pattern, engine: engine)
+    case engine
     when :dfa
-      @dfa.match?(input, use_cache?(input))
-    else
-      raise ArgumentError, "Unknown method: #{@method}"
+      compiled.match?(input, use_cache?(input))
+    when :vm
+      VM::Evaluator.evaluate(compiled, input, 0, 0)
     end
   end
 
-  private
-
   def use_cache?(input)
     input.length > 1000
   end
+  private_class_method :use_cache?
 end

data/spec/hoozuki/automaton/dfa/builder_spec.rb

@@ -0,0 +1,79 @@
+# frozen_string_literal: true
+
+RSpec.describe Hoozuki::Automaton::DFA::Builder do
+  describe '#call' do
+    let(:state) { Hoozuki::Automaton::StateID.new(0) }
+
+    context 'with simple NFA' do
+      it 'builds a DFA' do
+        node = Hoozuki::Node::Literal.new('a')
+        nfa = Hoozuki::Automaton::NFA.from_node(node, state)
+        builder = described_class.new(nfa, false)
+        dfa = builder.call
+
+        expect(dfa).to be_a(Hoozuki::Automaton::DFA)
+        expect(dfa.start).to be_a(Integer)
+        expect(dfa.accept).to be_a(Set)
+        expect(dfa.transitions).to be_a(Set)
+      end
+    end
+
+    context 'with NFA containing epsilon transitions' do
+      it 'eliminates epsilon transitions' do
+        node = Hoozuki::Parser.new.parse('a?')
+        nfa = Hoozuki::Automaton::NFA.from_node(node, state)
+        builder = described_class.new(nfa, false)
+        dfa = builder.call
+
+        epsilon_transitions = dfa.transitions.select { |_, label, _| label.nil? }
+        expect(epsilon_transitions).to be_empty
+      end
+    end
+
+
+    context 'with alternation' do
+      it 'creates correct number of states' do
+        node = Hoozuki::Parser.new.parse('a|b')
+        nfa = Hoozuki::Automaton::NFA.from_node(node, state)
+        builder = described_class.new(nfa, false)
+        dfa = builder.call
+
+        expect(dfa.transitions.size).to be >= 2
+      end
+    end
+
+    context 'with repetition' do
+      it 'handles loops correctly' do
+        node = Hoozuki::Parser.new.parse('a*')
+        nfa = Hoozuki::Automaton::NFA.from_node(node, state)
+        builder = described_class.new(nfa, false)
+        dfa = builder.call
+
+        expect(dfa.accept).to include(dfa.start)
+      end
+    end
+
+    context 'with concatenation' do
+      it 'builds sequential transitions' do
+        node = Hoozuki::Parser.new.parse('abc')
+        nfa = Hoozuki::Automaton::NFA.from_node(node, state)
+        builder = described_class.new(nfa, false)
+        dfa = builder.call
+
+        expect(dfa.transitions.size).to be >= 3
+      end
+    end
+
+    context 'with complex pattern' do
+      it 'builds correct DFA structure' do
+        node = Hoozuki::Parser.new.parse('(a|b)*c')
+        nfa = Hoozuki::Automaton::NFA.from_node(node, state)
+        builder = described_class.new(nfa, false)
+        dfa = builder.call
+
+        expect(dfa).to be_a(Hoozuki::Automaton::DFA)
+        expect(dfa.accept).not_to be_empty
+      end
+    end
+  end
+end

data/spec/hoozuki/automaton/dfa_spec.rb

@@ -0,0 +1,149 @@
+# frozen_string_literal: true
+
+RSpec.describe Hoozuki::Automaton::DFA do
+  describe '.from_nfa' do
+    let(:state) { Hoozuki::Automaton::StateID.new(0) }
+
+    it 'converts simple NFA to DFA' do
+      node = Hoozuki::Node::Literal.new('a')
+      nfa = Hoozuki::Automaton::NFA.from_node(node, state)
+      dfa = described_class.from_nfa(nfa, false)
+
+      expect(dfa).to be_a(described_class)
+      expect(dfa.start).to be_a(Integer)
+      expect(dfa.accept).to be_a(Set)
+      expect(dfa.transitions).not_to be_empty
+    end
+
+    it 'converts choice NFA to DFA' do
+      node = Hoozuki::Node::Choice.new([
+        Hoozuki::Node::Literal.new('a'),
+        Hoozuki::Node::Literal.new('b')
+      ])
+      nfa = Hoozuki::Automaton::NFA.from_node(node, state)
+      dfa = described_class.from_nfa(nfa, false)
+
+      expect(dfa.start).to be_a(Integer)
+      expect(dfa.transitions.map { |_, label, _| label }).to include('a', 'b')
+    end
+
+    it 'converts concatenation NFA to DFA' do
+      node = Hoozuki::Node::Concatenation.new([
+        Hoozuki::Node::Literal.new('a'),
+        Hoozuki::Node::Literal.new('b')
+      ])
+      nfa = Hoozuki::Automaton::NFA.from_node(node, state)
+      dfa = described_class.from_nfa(nfa, false)
+
+      expect(dfa.start).to be_a(Integer)
+      expect(dfa.accept).not_to be_empty
+    end
+
+    it 'handles alternation patterns' do
+      node = Hoozuki::Parser.new.parse('a|b')
+      nfa = Hoozuki::Automaton::NFA.from_node(node, state)
+      dfa = described_class.from_nfa(nfa, false)
+
+      expect(dfa.transitions.size).to be >= 2
+    end
+
+    it 'handles repetition patterns' do
+      node = Hoozuki::Parser.new.parse('a*')
+      nfa = Hoozuki::Automaton::NFA.from_node(node, state)
+      dfa = described_class.from_nfa(nfa, false)
+
+      expect(dfa.accept).to include(dfa.start)
+    end
+  end
+
+  describe '#match?' do
+    let(:state) { Hoozuki::Automaton::StateID.new(0) }
+
+    it 'matches using DFA for single literal' do
+      node = Hoozuki::Node::Literal.new('a')
+      nfa = Hoozuki::Automaton::NFA.from_node(node, state)
+      dfa = described_class.from_nfa(nfa, false)
+
+      expect(dfa.match?('a', false)).to be true
+      expect(dfa.match?('b', false)).to be false
+    end
+
+    it 'matches choice pattern using DFA' do
+      node = Hoozuki::Node::Choice.new([
+        Hoozuki::Node::Literal.new('a'),
+        Hoozuki::Node::Literal.new('b')
+      ])
+      nfa = Hoozuki::Automaton::NFA.from_node(node, state)
+      dfa = described_class.from_nfa(nfa, false)
+
+      expect(dfa.match?('a', false)).to be true
+      expect(dfa.match?('b', false)).to be true
+      expect(dfa.match?('c', false)).to be false
+    end
+
+    it 'matches concatenation pattern using DFA' do
+      node = Hoozuki::Node::Concatenation.new([
+        Hoozuki::Node::Literal.new('a'),
+        Hoozuki::Node::Literal.new('b')
+      ])
+      nfa = Hoozuki::Automaton::NFA.from_node(node, state)
+      dfa = described_class.from_nfa(nfa, false)
+
+      expect(dfa.match?('ab', false)).to be true
+      expect(dfa.match?('a', false)).to be false
+      expect(dfa.match?('abc', false)).to be false
+    end
+
+    context 'with simple literal' do
+      it 'matches exact string' do
+        node = Hoozuki::Parser.new.parse('abc')
+        nfa = Hoozuki::Automaton::NFA.from_node(node, state)
+        dfa = described_class.from_nfa(nfa, false)
+
+        expect(dfa.match?('abc', false)).to be true
+      end
+
+      it 'does not match different string' do
+        node = Hoozuki::Parser.new.parse('abc')
+        nfa = Hoozuki::Automaton::NFA.from_node(node, state)
+        dfa = described_class.from_nfa(nfa, false)
+
+        expect(dfa.match?('abd', false)).to be false
+      end
+    end
+
+    context 'with alternation' do
+      it 'matches either branch' do
+        node = Hoozuki::Parser.new.parse('a|b')
+        nfa = Hoozuki::Automaton::NFA.from_node(node, state)
+        dfa = described_class.from_nfa(nfa, false)
+
+        expect(dfa.match?('a', false)).to be true
+        expect(dfa.match?('b', false)).to be true
+        expect(dfa.match?('c', false)).to be false
+      end
+    end
+  end
+
+  describe '#next_transition' do
+    let(:state) { Hoozuki::Automaton::StateID.new(0) }
+
+    it 'finds correct next state' do
+      node = Hoozuki::Parser.new.parse('a')
+      nfa = Hoozuki::Automaton::NFA.from_node(node, state)
+      dfa = described_class.from_nfa(nfa, false)
+
+      next_state = dfa.next_transition(dfa.start, 'a', false)
+      expect(next_state).not_to be_nil
+    end
+
+    it 'returns nil for invalid transition' do
+      node = Hoozuki::Parser.new.parse('a')
+      nfa = Hoozuki::Automaton::NFA.from_node(node, state)
+      dfa = described_class.from_nfa(nfa, false)
+
+      next_state = dfa.next_transition(dfa.start, 'b', false)
+      expect(next_state).to be_nil
+    end
+  end
+end