hammer-parser 0.2.1

data/lib/hammer/parser.rb

@@ -0,0 +1,224 @@
+require 'hammer/internal'
+
+module Hammer
+  class Parser
+
+    @@saved_objects = Hammer::Internal::DynamicVariable.new nil, "Hammer parse-time pins"
+
+    # Don't create new instances with Hammer::Parser.new,
+    # use the constructor methods instead (i.e. Hammer::Parser.int64 etc.)
+    #
+    # name: Name of the parser. Should be a symbol.
+    # h_parser: The pointer to the parser as returned by hammer.
+    # dont_gc: Pass additional data that's used by the parser and needs to be saved from the garbage collector (at least as long this object lives).
+    def initialize(name, h_parser, dont_gc=[])
+      @name = name
+      @h_parser = h_parser
+      # Always store as array, so we can easily add stuff later on
+      dont_gc = [dont_gc] unless dont_gc.is_a? Array
+      @dont_gc = dont_gc.dup
+    end
+
+    # dont_gc is required to build a fuzzer from the declaration of Hammer::Parser object.
+    attr_reader :name
+    attr_reader :h_parser
+    attr_reader :dont_gc
+
+    # Parse the given data. Returns the parse result if successful, nil otherwise.
+    #
+    # data: A string containing the data to parse.
+    def parse(data)
+      raise RuntimeError, '@h_parser is nil' if @h_parser.nil?
+      raise ArgumentError, 'expecting a String' unless data.is_a? String # TODO: Not needed, FFI checks that.
+
+      ibuf = FFI::MemoryPointer.from_string(data)
+      @@saved_objects.with([]) do
+          result = Hammer::Internal.h_parse(@h_parser, ibuf, data.bytesize) # Don't include the trailing null
+          if result.null?
+            return nil
+          else
+            # NOTE:
+            # The parse result *must* hold a reference to the parser that created it!
+            # Otherwise, the parser might get garbage-collected while the result is still valid.
+            # Any pointers to token strings will then be invalid.
+            result.instance_variable_set :@parser, self
+            result.instance_variable_set :@pins, @@saved_objects.value
+            return result
+          end
+        end
+    end
+
+    # Binds an indirect parser.
+    def bind(other_parser)
+      raise RuntimeError, 'can only bind indirect parsers' unless self.name == :indirect
+      Hammer::Internal.h_bind_indirect(self.h_parser, other_parser.h_parser)
+      @dont_gc << other_parser
+    end
+
+    # Can pass the action either as a Proc in second parameter, or as block.
+    def self.action(parser, action=nil, &block)
+      action = block if action.nil?
+      raise ArgumentError, 'no action' if action.nil?
+
+      real_action = Proc.new {|hpr|
+                          ret = action.call(hpr.ast)
+                          # Pin the result
+                          @@saved_objects.value << ret
+                          hpt = hpr.arena_alloc(Hammer::Internal::HParsedToken)
+                          unless hpr.ast.nil?
+                            hpt[:index] = hpr[:ast][:index]
+                            hpt[:bit_offset] = hpr[:ast][:bit_offset]
+                          end
+                          hpt[:token_type] = :"com.upstandinghackers.hammer.ruby.object"
+                          hpt[:data][:uint] = ret.object_id
+                          hpt
+                        }
+
+      h_parser = Hammer::Internal.h_action(parser.h_parser, real_action)
+      return Hammer::Parser.new(:action, h_parser, [parser, action, real_action])
+    end
+
+    # Can pass the predicate either as a Proc in second parameter, or as block.
+    def self.attr_bool(parser, predicate=nil, &block)
+      predicate = block if predicate.nil?
+      raise ArgumentError, 'no predicate' if predicate.nil?
+
+      real_pred = Proc.new {|hpr| predicate.call hpr.ast}
+
+      h_parser = Hammer::Internal.h_attr_bool(parser.h_parser, real_pred)
+      return Hammer::Parser.new(:attr_bool, h_parser, [parser, predicate, real_pred])
+    end
+
+    def self.token(string)
+      # Need to copy string to a memory buffer (not just string.dup)
+      # * Original string might be modified, this must not affect existing tokens
+      # * We need a constant memory address (Ruby string might be moved around by the Ruby VM)
+      buffer = FFI::MemoryPointer.from_string(string)
+      h_parser = Hammer::Internal.h_token(buffer, buffer.size-1) # buffer.size includes the null byte at the end
+      encoding = string.encoding
+
+      wrapping_action = Proc.new {|hpr|
+                              hstr = hpr.arena_alloc(Hammer::Internal::HString)
+                              hstr[:content] = hpr[:ast][:data][:bytes]
+                              hstr[:encoding] = encoding.object_id
+
+                              hpt = hpr.arena_alloc(Hammer::Internal::HParsedToken)
+                              hpt[:token_type] = :"com.upstandinghackers.hammer.ruby.encodedStr"
+                              hpt[:data][:user] = hstr.to_ptr
+                              hpt[:bit_offset] = hpr[:ast][:bit_offset]
+                              hpt[:index] = hpr[:ast][:index]
+                              hpt
+                            }
+      wrapped_parser = Hammer::Internal.h_action(h_parser, wrapping_action)
+      return Hammer::Parser.new(:token, wrapped_parser, [buffer, string, encoding, wrapping_action, h_parser])
+    end
+
+    def self.marshal_ch_arg(num)
+      if num.is_a?(String)
+        raise ArgumentError, "Expecting either a fixnum in 0..255 or a single-byte String" unless num.bytesize == 1
+        num = num.bytes.first
+      end
+      raise ArgumentError, 'Expecting a Fixnum in 0..255 or a single-byte String' unless num.is_a?(Fixnum) and num.between?(0, 255)
+      return num
+    end
+    private_class_method :marshal_ch_arg
+
+    def self.ch_parser_wrapper(parser)
+      return Hammer::Parser.action(parser) {|x| x.data.chr}
+    end
+
+    def self.ch(ch)
+      num = marshal_ch_arg(ch)
+      h_parser = Hammer::Internal.h_ch(num)
+
+      return ch_parser_wrapper(Hammer::Parser.new(:ch, h_parser, nil))
+    end
+
+    def self.ch_range(ch1, ch2)
+      ch1 = marshal_ch_arg(ch1)
+      ch2 = marshal_ch_arg(ch2)
+      h_parser = Hammer::Internal.h_ch_range(ch1, ch2)
+      return ch_parser_wrapper(Hammer::Parser.new(:ch_range, h_parser, nil))
+    end
+
+    def self.int_range(parser, i1, i2)
+      h_parser = Hammer::Internal.h_int_range(parser.h_parser, i1, i2)
+      return Hammer::Parser.new(:int_range, h_parser, [parser])
+    end
+
+    def self.in(charset)
+      raise ArgumentError, "Expected a String" unless charset.is_a?(String)
+      ibuf = FFI::MemoryPointer.from_string(charset)
+      h_parser = Hammer::Internal.h_in(ibuf, charset.bytesize)
+      return ch_parser_wrapper(Hammer::Parser.new(:in, h_parser, nil))
+    end
+
+    def self.repeat_n(parser, count)
+      h_parser = Hammer::Internal.h_repeat_n(parser.h_parser, count)
+      return Hammer::Parser.new(:repeat_n, h_parser, nil)
+    end
+
+    def self.not_in(charset)
+      raise ArgumentError, "Expected a String" unless charset.is_a?(String)
+      ibuf = FFI::MemoryPointer.from_string(charset)
+      h_parser = Hammer::Internal.h_not_in(ibuf, charset.bytesize)
+      return ch_parser_wrapper(Hammer::Parser.new(:not_in, h_parser, nil))
+    end
+
+    # Defines a parser constructor with the given name.
+    # Options:
+    #   hammer_function: name of the hammer function to call (default: 'h_'+name)
+    #   varargs: Whether the function is taking a variable number of arguments (default: false)
+    def self.define_parser(name, options = {})
+      hammer_function = options[:hammer_function] || ('h_' + name.to_s).to_sym
+      varargs = options[:varargs] || false
+
+      # Define a new class method
+      define_singleton_method name do |*parsers|
+        if varargs
+          args = parsers.flat_map { |p| [:pointer, p.h_parser] }
+          args += [:pointer, nil]
+        else
+          args = parsers.map(&:h_parser)
+        end
+        h_parser = Hammer::Internal.send hammer_function, *args
+
+        return Hammer::Parser.new(name, h_parser, parsers)
+      end
+    end
+    private_class_method :define_parser
+
+    define_parser :sequence, varargs: true
+    define_parser :choice, varargs: true
+
+    define_parser :int64
+    define_parser :int32
+    define_parser :int16
+    define_parser :int8
+    define_parser :uint64
+    define_parser :uint32
+    define_parser :uint16
+    define_parser :uint8
+    define_parser :whitespace
+    define_parser :left
+    define_parser :right
+    define_parser :middle
+    define_parser :end_p
+    define_parser :nothing_p
+    define_parser :butnot
+    define_parser :difference
+    define_parser :xor
+    define_parser :many
+    define_parser :many1
+    define_parser :optional
+    define_parser :ignore
+    define_parser :sepBy
+    define_parser :sepBy1
+    define_parser :epsilon_p
+    define_parser :length_value
+    define_parser :and
+    define_parser :not
+    define_parser :indirect
+
+  end
+end

data/lib/hammer/parser_builder.rb

@@ -0,0 +1,124 @@
+# TODO: Find a way to make docile an optional dependency
+# (autoload for this file? and throw some informative error when docile isn't available.
+# should also check gem version with a 'gem' call and appropriate version specifier.)
+require 'docile'
+
+module Hammer
+
+  class Parser
+    def self.build(&block)
+      ParserBuilder.new.sequence(&block).build
+    end
+
+    def self.build_choice(&block)
+      ParserBuilder.new.choice(&block).build
+    end
+  end # class Parser
+
+  class ParserBuilder
+    attr_reader :parsers
+
+    def initialize
+      @parsers = []
+    end
+
+    def build
+      if @parsers.length > 1
+        Hammer::Parser.sequence(*@parsers)
+      else
+        @parsers.first
+      end
+    end
+
+
+    # can call it either as ParserBuiler.new.sequence(parser1, parser2, parser3)
+    # or as Parser.build { sequence { call parser1; call parser2; call parser3 } }
+    def sequence(*parsers, &block)
+      @parsers += parsers
+      @parsers << Docile.dsl_eval(ParserBuilder.new, &block).build if block_given?
+      return self
+    end
+
+    def choice(*parsers, &block)
+      if block_given?
+        parsers += Docile.dsl_eval(ParserBuilder.new, &block).parsers
+      end
+      @parsers << Hammer::Parser.choice(*parsers)
+      return self
+    end
+
+    def call(parser)
+      @parsers << parser
+      return self
+    end
+
+    # modifies previous parser
+    def action(&block)
+      parser = @parsers.last
+      raise RuntimeError, 'need a parser before action' if parser.nil?
+      @parsers << Hammer::Parser.action(parser, &block)
+      return self
+    end
+
+    # Defines a parser constructor with the given name.
+    def self.define_parser(name, options = {})
+      define_method name do |*args|
+        # TODO: This is wrong!! Needs to accept a block for nested parsers!
+        @parsers << Hammer::Parser.send(name, *args)
+        return self
+      end
+    end
+    private_class_method :define_parser
+
+    define_parser :token
+    define_parser :ch
+    define_parser :int64
+    define_parser :int32
+    define_parser :int16
+    define_parser :int8
+    define_parser :uint64
+    define_parser :uint32
+    define_parser :uint16
+    define_parser :uint8
+    define_parser :whitespace
+    define_parser :left
+    define_parser :right
+    define_parser :middle
+    define_parser :end_p
+    define_parser :nothing_p
+    define_parser :butnot
+    define_parser :difference
+    define_parser :xor
+    define_parser :many
+    define_parser :many1
+    define_parser :optional
+    define_parser :ignore
+    define_parser :sepBy
+    define_parser :sepBy1
+    define_parser :epsilon_p
+    define_parser :length_value
+    define_parser :and
+    define_parser :not
+
+    # At least indirect must return the parser instead of the builder, so it can be stored in a variable.
+    # Other possible solution:
+    #   Make indirect take a name parameter, and use the name to bind it later.
+    #   Example:
+    #     p = Hammer::Parser.build { indirect(:the_name) }
+    #     p.bind(:the_name, inner_parser)
+    #   (store names and parsers in hash in the builder,
+    #    when building merge hashes from sub builders and store everything in the resulting sequence or choice.
+    #    make Parser#bind take and optional symbol. if it is given, the name is looked up in the table.)
+    # TODO:
+    #   Think about this more.
+    #   Do we need to be able to build parsers by chaining function calls? DSL should be sufficient.
+    #   If yes, the parser methods in this class should not return "self", but the Hammer::Parser object they create.
+    def indirect
+      parser = Hammer::Parser.indirect
+      @parsers << parser
+      return parser
+    end
+
+  end # class ParserBuilder
+
+end # module Hammer

data/lib/minitest/hamer-parser_plugin.rb

@@ -0,0 +1,31 @@
+module Minitest
+
+  module Assertions
+    HAMMER_JUST_PARSE = Object.new
+    def assert_parse_ok(parser, probe, expected=HAMMER_JUST_PARSE)
+      refute_nil parser, "Parser must not be nil (this is a problem with your test)"
+      parse_result = parser.parse(probe)
+      refute_nil parse_result, "Parse failed"
+      if HAMMER_JUST_PARSE != expected
+        if parse_result.ast == nil
+          assert_nil expected, "Parser returned nil AST; expected #{expected}"
+        else
+          assert_equal parse_result.ast.unmarshal, expected
+        end
+      end
+    end
+
+    def refute_parse_ok(parser, probe)
+      refute_nil parser, "Parser must not be nil (this is a problem with your test)"
+      parse_result = parser.parse(probe)
+
+      if not parse_result.nil?
+        assert_nil parse_result, "Parse succeeded unexpectedly with " + parse_result.ast.inspect
+      end
+    end
+  end
+  
+  
+ #def self.plugin_hammer-parser_init(options)
+end
+    

data/test/autogen_test.rb

@@ -0,0 +1,755 @@
+require 'bundler/setup'
+require 'minitest/autorun'
+require 'hammer-parser'
+class TestToken < Minitest::Test
+  def setup
+    super
+    h = Hammer::Parser
+    @parser_1 = h.token("95\xa2")
+  end
+
+  def test_1
+    assert_parse_ok @parser_1, "95\xa2", "95\xa2"
+  end
+
+  def test_2
+    refute_parse_ok @parser_1, "95\xa3"
+  end
+end
+
+class TestCh < Minitest::Test
+  def setup
+    super
+    h = Hammer::Parser
+    @parser_1 = h.ch(0xa2)
+  end
+
+  def test_1
+    assert_parse_ok @parser_1, "\xa2", 0xa2.chr
+  end
+
+  def test_2
+    refute_parse_ok @parser_1, "\xa3"
+  end
+end
+
+class TestChRange < Minitest::Test
+  def setup
+    super
+    h = Hammer::Parser
+    @parser_1 = h.ch_range(0x61, 0x63)
+  end
+
+  def test_1
+    assert_parse_ok @parser_1, "b", 0x62.chr
+  end
+
+  def test_2
+    refute_parse_ok @parser_1, "d"
+  end
+end
+
+class TestInt64 < Minitest::Test
+  def setup
+    super
+    h = Hammer::Parser
+    @parser_1 = h.int64
+  end
+
+  def test_1
+    assert_parse_ok @parser_1, "\xff\xff\xff\xfe\x00\x00\x00\x00", -0x200000000
+  end
+
+  def test_2
+    refute_parse_ok @parser_1, "\xff\xff\xff\xfe\x00\x00\x00"
+  end
+end
+
+class TestInt32 < Minitest::Test
+  def setup
+    super
+    h = Hammer::Parser
+    @parser_1 = h.int32
+  end
+
+  def test_1
+    assert_parse_ok @parser_1, "\xff\xfe\x00\x00", -0x20000
+  end
+
+  def test_2
+    refute_parse_ok @parser_1, "\xff\xfe\x00"
+  end
+
+  def test_3
+    assert_parse_ok @parser_1, "\x00\x02\x00\x00", 0x20000
+  end
+
+  def test_4
+    refute_parse_ok @parser_1, "\x00\x02\x00"
+  end
+end
+
+class TestInt16 < Minitest::Test
+  def setup
+    super
+    h = Hammer::Parser
+    @parser_1 = h.int16
+  end
+
+  def test_1
+    assert_parse_ok @parser_1, "\xfe\x00", -0x200
+  end
+
+  def test_2
+    refute_parse_ok @parser_1, "\xfe"
+  end
+
+  def test_3
+    assert_parse_ok @parser_1, "\x02\x00", 0x200
+  end
+
+  def test_4
+    refute_parse_ok @parser_1, "\x02"
+  end
+end
+
+class TestInt8 < Minitest::Test
+  def setup
+    super
+    h = Hammer::Parser
+    @parser_1 = h.int8
+  end
+
+  def test_1
+    assert_parse_ok @parser_1, "\x88", -0x78
+  end
+
+  def test_2
+    refute_parse_ok @parser_1, ""
+  end
+end
+
+class TestUint64 < Minitest::Test
+  def setup
+    super
+    h = Hammer::Parser
+    @parser_1 = h.uint64
+  end
+
+  def test_1
+    assert_parse_ok @parser_1, "\x00\x00\x00\x02\x00\x00\x00\x00", 0x200000000
+  end
+
+  def test_2
+    refute_parse_ok @parser_1, "\x00\x00\x00\x02\x00\x00\x00"
+  end
+end
+
+class TestUint32 < Minitest::Test
+  def setup
+    super
+    h = Hammer::Parser
+    @parser_1 = h.uint32
+  end
+
+  def test_1
+    assert_parse_ok @parser_1, "\x00\x02\x00\x00", 0x20000
+  end
+
+  def test_2
+    refute_parse_ok @parser_1, "\x00\x02\x00"
+  end
+end
+
+class TestUint16 < Minitest::Test
+  def setup
+    super
+    h = Hammer::Parser
+    @parser_1 = h.uint16
+  end
+
+  def test_1
+    assert_parse_ok @parser_1, "\x02\x00", 0x200
+  end
+
+  def test_2
+    refute_parse_ok @parser_1, "\x02"
+  end
+end
+
+class TestUint8 < Minitest::Test
+  def setup
+    super
+    h = Hammer::Parser
+    @parser_1 = h.uint8
+  end
+
+  def test_1
+    assert_parse_ok @parser_1, "x", 0x78
+  end
+
+  def test_2
+    refute_parse_ok @parser_1, ""
+  end
+end
+
+class TestIntRange < Minitest::Test
+  def setup
+    super
+    h = Hammer::Parser
+    @parser_1 = h.int_range(h.uint8, 0x3, 0xa)
+  end
+
+  def test_1
+    assert_parse_ok @parser_1, "\x05", 0x5
+  end
+
+  def test_2
+    refute_parse_ok @parser_1, "\x0b"
+  end
+end
+
+class TestWhitespace < Minitest::Test
+  def setup
+    super
+    h = Hammer::Parser
+    @parser_1 = h.whitespace(h.ch(0x61))
+    @parser_2 = h.whitespace(h.end_p)
+  end
+
+  def test_1
+    assert_parse_ok @parser_1, "a", 0x61.chr
+  end
+
+  def test_2
+    assert_parse_ok @parser_1, " a", 0x61.chr
+  end
+
+  def test_3
+    assert_parse_ok @parser_1, "  a", 0x61.chr
+  end
+
+  def test_4
+    assert_parse_ok @parser_1, "\x09a", 0x61.chr
+  end
+
+  def test_5
+    refute_parse_ok @parser_1, "_a"
+  end
+
+  def test_6
+    assert_parse_ok @parser_2, "", nil
+  end
+
+  def test_7
+    assert_parse_ok @parser_2, "  ", nil
+  end
+
+  def test_8
+    refute_parse_ok @parser_2, "  x"
+  end
+end
+
+class TestLeft < Minitest::Test
+  def setup
+    super
+    h = Hammer::Parser
+    @parser_1 = h.left(h.ch(0x61), h.ch(0x20))
+  end
+
+  def test_1
+    assert_parse_ok @parser_1, "a ", 0x61.chr
+  end
+
+  def test_2
+    refute_parse_ok @parser_1, "a"
+  end
+
+  def test_3
+    refute_parse_ok @parser_1, " "
+  end
+
+  def test_4
+    refute_parse_ok @parser_1, "ba"
+  end
+end
+
+class TestMiddle < Minitest::Test
+  def setup
+    super
+    h = Hammer::Parser
+    @parser_1 = h.middle(h.ch(0x20.chr), h.ch(0x61.chr), h.ch(0x20.chr))
+  end
+
+  def test_1
+    assert_parse_ok @parser_1, " a ", 0x61.chr
+  end
+
+  def test_2
+    refute_parse_ok @parser_1, "a"
+  end
+
+  def test_3
+    refute_parse_ok @parser_1, " a"
+  end
+
+  def test_4
+    refute_parse_ok @parser_1, "a "
+  end
+
+  def test_5
+    refute_parse_ok @parser_1, " b "
+  end
+
+  def test_6
+    refute_parse_ok @parser_1, "ba "
+  end
+
+  def test_7
+    refute_parse_ok @parser_1, " ab"
+  end
+end
+
+class TestIn < Minitest::Test
+  def setup
+    super
+    h = Hammer::Parser
+    @parser_1 = h.in("abc")
+  end
+
+  def test_1
+    assert_parse_ok @parser_1, "b", 0x62.chr
+  end
+
+  def test_2
+    refute_parse_ok @parser_1, "d"
+  end
+end
+
+class TestNotIn < Minitest::Test
+  def setup
+    super
+    h = Hammer::Parser
+    @parser_1 = h.not_in("abc")
+  end
+
+  def test_1
+    assert_parse_ok @parser_1, "d", 0x64.chr
+  end
+
+  def test_2
+    refute_parse_ok @parser_1, "a"
+  end
+end
+
+class TestEndP < Minitest::Test
+  def setup
+    super
+    h = Hammer::Parser
+    @parser_1 = h.sequence(h.ch(0x61.chr), h.end_p)
+  end
+
+  def test_1
+    assert_parse_ok @parser_1, "a", [0x61.chr]
+  end
+
+  def test_2
+    refute_parse_ok @parser_1, "aa"
+  end
+end
+
+class TestNothingP < Minitest::Test
+  def setup
+    super
+    h = Hammer::Parser
+    @parser_1 = h.nothing_p
+  end
+
+  def test_1
+    refute_parse_ok @parser_1, "a"
+  end
+end
+
+class TestSequence < Minitest::Test
+  def setup
+    super
+    h = Hammer::Parser
+    @parser_1 = h.sequence(h.ch(0x61.chr), h.ch(0x62.chr))
+    @parser_2 = h.sequence(h.ch(0x61.chr), h.whitespace(h.ch(0x62.chr)))
+  end
+
+  def test_1
+    assert_parse_ok @parser_1, "ab", [0x61.chr, 0x62.chr]
+  end
+
+  def test_2
+    refute_parse_ok @parser_1, "a"
+  end
+
+  def test_3
+    refute_parse_ok @parser_1, "b"
+  end
+
+  def test_4
+    assert_parse_ok @parser_2, "ab", [0x61.chr, 0x62.chr]
+  end
+
+  def test_5
+    assert_parse_ok @parser_2, "a b", [0x61.chr, 0x62.chr]
+  end
+
+  def test_6
+    assert_parse_ok @parser_2, "a  b", [0x61.chr, 0x62.chr]
+  end
+end
+
+class TestChoice < Minitest::Test
+  def setup
+    super
+    h = Hammer::Parser
+    @parser_1 = h.choice(h.ch(0x61.chr), h.ch(0x62.chr))
+  end
+
+  def test_1
+    assert_parse_ok @parser_1, "a", 0x61.chr
+  end
+
+  def test_2
+    assert_parse_ok @parser_1, "b", 0x62.chr
+  end
+
+  def test_3
+    assert_parse_ok @parser_1, "ab", 0x61.chr
+  end
+
+  def test_4
+    refute_parse_ok @parser_1, "c"
+  end
+end
+
+class TestButnot < Minitest::Test
+  def setup
+    super
+    h = Hammer::Parser
+    @parser_1 = h.butnot(h.ch(0x61.chr), h.token("ab"))
+    @parser_2 = h.butnot(h.ch_range(0x30.chr, 0x39.chr), h.ch(0x36.chr))
+  end
+
+  def test_1
+    assert_parse_ok @parser_1, "a", 0x61.chr
+  end
+
+  def test_2
+    refute_parse_ok @parser_1, "ab"
+  end
+
+  def test_3
+    assert_parse_ok @parser_1, "aa", 0x61.chr
+  end
+
+  def test_4
+    assert_parse_ok @parser_2, "5", 0x35.chr
+  end
+
+  def test_5
+    refute_parse_ok @parser_2, "6"
+  end
+end
+
+class TestDifference < Minitest::Test
+  def setup
+    super
+    h = Hammer::Parser
+    @parser_1 = h.difference(h.token("ab"), h.ch(0x61.chr))
+  end
+
+  def test_1
+    assert_parse_ok @parser_1, "ab", "ab"
+  end
+
+  def test_2
+    refute_parse_ok @parser_1, "a"
+  end
+end
+
+class TestXor < Minitest::Test
+  def setup
+    super
+    h = Hammer::Parser
+    @parser_1 = h.xor(h.ch_range(0x30.chr, 0x36.chr), h.ch_range(0x35.chr, 0x39.chr))
+  end
+
+  def test_1
+    assert_parse_ok @parser_1, "0", 0x30.chr
+  end
+
+  def test_2
+    assert_parse_ok @parser_1, "9", 0x39.chr
+  end
+
+  def test_3
+    refute_parse_ok @parser_1, "5"
+  end
+
+  def test_4
+    refute_parse_ok @parser_1, "a"
+  end
+end
+
+class TestMany < Minitest::Test
+  def setup
+    super
+    h = Hammer::Parser
+    @parser_1 = h.many(h.choice(h.ch(0x61.chr), h.ch(0x62.chr)))
+  end
+
+  def test_1
+    assert_parse_ok @parser_1, "", []
+  end
+
+  def test_2
+    assert_parse_ok @parser_1, "a", [0x61.chr]
+  end
+
+  def test_3
+    assert_parse_ok @parser_1, "b", [0x62.chr]
+  end
+
+  def test_4
+    assert_parse_ok @parser_1, "aabbaba", [0x61.chr, 0x61.chr, 0x62.chr, 0x62.chr, 0x61.chr, 0x62.chr, 0x61.chr]
+  end
+end
+
+class TestMany1 < Minitest::Test
+  def setup
+    super
+    h = Hammer::Parser
+    @parser_1 = h.many1(h.choice(h.ch(0x61.chr), h.ch(0x62.chr)))
+  end
+
+  def test_1
+    refute_parse_ok @parser_1, ""
+  end
+
+  def test_2
+    assert_parse_ok @parser_1, "a", [0x61.chr]
+  end
+
+  def test_3
+    assert_parse_ok @parser_1, "b", [0x62.chr]
+  end
+
+  def test_4
+    assert_parse_ok @parser_1, "aabbaba", [0x61.chr, 0x61.chr, 0x62.chr, 0x62.chr, 0x61.chr, 0x62.chr, 0x61.chr]
+  end
+
+  def test_5
+    refute_parse_ok @parser_1, "daabbabadef"
+  end
+end
+
+class TestRepeatN < Minitest::Test
+  def setup
+    super
+    h = Hammer::Parser
+    @parser_1 = h.repeat_n(h.choice(h.ch(0x61.chr), h.ch(0x62.chr)), 0x2)
+  end
+
+  def test_1
+    refute_parse_ok @parser_1, "adef"
+  end
+
+  def test_2
+    assert_parse_ok @parser_1, "abdef", [0x61.chr, 0x62.chr]
+  end
+
+  def test_3
+    refute_parse_ok @parser_1, "dabdef"
+  end
+end
+
+class TestOptional < Minitest::Test
+  def setup
+    super
+    h = Hammer::Parser
+    @parser_1 = h.sequence(h.ch(0x61.chr), h.optional(h.choice(h.ch(0x62.chr), h.ch(0x63.chr))), h.ch(0x64.chr))
+  end
+
+  def test_1
+    assert_parse_ok @parser_1, "abd", [0x61.chr, 0x62.chr, 0x64.chr]
+  end
+
+  def test_2
+    assert_parse_ok @parser_1, "acd", [0x61.chr, 0x63.chr, 0x64.chr]
+  end
+
+  def test_3
+    assert_parse_ok @parser_1, "ad", [0x61.chr, nil, 0x64.chr]
+  end
+
+  def test_4
+    refute_parse_ok @parser_1, "aed"
+  end
+
+  def test_5
+    refute_parse_ok @parser_1, "ab"
+  end
+
+  def test_6
+    refute_parse_ok @parser_1, "ac"
+  end
+end
+
+class TestIgnore < Minitest::Test
+  def setup
+    super
+    h = Hammer::Parser
+    @parser_1 = h.sequence(h.ch(0x61.chr), h.ignore(h.ch(0x62.chr)), h.ch(0x63.chr))
+  end
+
+  def test_1
+    assert_parse_ok @parser_1, "abc", [0x61.chr, 0x63.chr]
+  end
+
+  def test_2
+    refute_parse_ok @parser_1, "ac"
+  end
+end
+
+class TestSepBy < Minitest::Test
+  def setup
+    super
+    h = Hammer::Parser
+    @parser_1 = h.sepBy(h.choice(h.ch(0x31.chr), h.ch(0x32.chr), h.ch(0x33.chr)), h.ch(0x2c.chr))
+  end
+
+  def test_1
+    assert_parse_ok @parser_1, "1,2,3", [0x31.chr, 0x32.chr, 0x33.chr]
+  end
+
+  def test_2
+    assert_parse_ok @parser_1, "1,3,2", [0x31.chr, 0x33.chr, 0x32.chr]
+  end
+
+  def test_3
+    assert_parse_ok @parser_1, "1,3", [0x31.chr, 0x33.chr]
+  end
+
+  def test_4
+    assert_parse_ok @parser_1, "3", [0x33.chr]
+  end
+
+  def test_5
+    assert_parse_ok @parser_1, "", []
+  end
+end
+
+class TestSepBy1 < Minitest::Test
+  def setup
+    super
+    h = Hammer::Parser
+    @parser_1 = h.sepBy1(h.choice(h.ch(0x31.chr), h.ch(0x32.chr), h.ch(0x33.chr)), h.ch(0x2c.chr))
+  end
+
+  def test_1
+    assert_parse_ok @parser_1, "1,2,3", [0x31.chr, 0x32.chr, 0x33.chr]
+  end
+
+  def test_2
+    assert_parse_ok @parser_1, "1,3,2", [0x31.chr, 0x33.chr, 0x32.chr]
+  end
+
+  def test_3
+    assert_parse_ok @parser_1, "1,3", [0x31.chr, 0x33.chr]
+  end
+
+  def test_4
+    assert_parse_ok @parser_1, "3", [0x33.chr]
+  end
+
+  def test_5
+    refute_parse_ok @parser_1, ""
+  end
+end
+
+class TestAnd < Minitest::Test
+  def setup
+    super
+    h = Hammer::Parser
+    @parser_1 = h.sequence(h.and(h.ch(0x30.chr)), h.ch(0x30.chr))
+    @parser_2 = h.sequence(h.and(h.ch(0x30.chr)), h.ch(0x31.chr))
+    @parser_3 = h.sequence(h.ch(0x31.chr), h.and(h.ch(0x32.chr)))
+  end
+
+  def test_1
+    assert_parse_ok @parser_1, "0", [0x30.chr]
+  end
+
+  def test_2
+    refute_parse_ok @parser_1, "1"
+  end
+
+  def test_3
+    refute_parse_ok @parser_2, "0"
+  end
+
+  def test_4
+    refute_parse_ok @parser_2, "1"
+  end
+
+  def test_5
+    assert_parse_ok @parser_3, "12", [0x31.chr]
+  end
+
+  def test_6
+    refute_parse_ok @parser_3, "13"
+  end
+end
+
+class TestNot < Minitest::Test
+  def setup
+    super
+    h = Hammer::Parser
+    @parser_1 = h.sequence(h.ch(0x61.chr), h.choice(h.token("+"), h.token("++")), h.ch(0x62.chr))
+    @parser_2 = h.sequence(h.ch(0x61.chr), h.choice(h.sequence(h.token("+"), h.not(h.ch(0x2b.chr))), h.token("++")), h.ch(0x62.chr))
+  end
+
+  def test_1
+    assert_parse_ok @parser_1, "a+b", [0x61.chr, "+", 0x62.chr]
+  end
+
+  def test_2
+    refute_parse_ok @parser_1, "a++b"
+  end
+
+  def test_3
+    assert_parse_ok @parser_2, "a+b", [0x61.chr, ["+"], 0x62.chr]
+  end
+
+  def test_4
+    assert_parse_ok @parser_2, "a++b", [0x61.chr, "++", 0x62.chr]
+  end
+end
+
+class TestRightrec < Minitest::Test
+  def setup
+    super
+    h = Hammer::Parser
+    @sp_rr = h.indirect
+    @sp_rr.bind h.choice(h.sequence(h.ch(0x61.chr), @sp_rr), h.epsilon_p)
+    @parser_1 = @sp_rr
+  end
+
+  def test_1
+    assert_parse_ok @parser_1, "a", [0x61.chr]
+  end
+
+  def test_2
+    assert_parse_ok @parser_1, "aa", [0x61.chr, [0x61.chr]]
+  end
+
+  def test_3
+    assert_parse_ok @parser_1, "aaa", [0x61.chr, [0x61.chr, [0x61.chr]]]
+  end
+end
+