hammer-parser 0.2.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: b813026f979c544803273ae966bd9f1858e4f510
+  data.tar.gz: b48b3e11a02fc19be47139330a2a14e7c91558f6
+SHA512:
+  metadata.gz: d33df5b0e64a59baf7d27ce600edfca7eabf6055fea496b4df5ac148cf036183d88959ac3c3a7d7bc56f23273e132d2409b308cc61177925536a914f7e317b16
+  data.tar.gz: 147863e19142b07d561f44f9ccd6b994d8553c7cd2ef7cf5b6d03b7507ffaa95d0aa76fc63d3d3d438397a599bcee124fbb691478548de7acc7d3765f8a72ca3

data/README.md

@@ -0,0 +1,101 @@
+# hammer-parser
+
+Ruby bindings for [hammer](https://github.com/UpstandingHackers/hammer), a parsing library.
+
+
+## Notes
+
+* I called the gem `hammer-parser`, since there already is a [gem named `hammer`](https://rubygems.org/gems/hammer).
+
+
+## Development
+
+1. `cd src/bindings/ruby`.
+
+2. Run `bundle install` to install dependencies.
+
+3. Run `bundle console` to open `irb` with hammer loaded.
+
+4. To run tests, just run `bundle exec rake test`.
+
+
+## Installation
+
+1. Download the hammer source code, and make it available system wide with the bindings. 
+  
+  `git clone https://github.com/UpstandingHackers/hammer`
+
+  `cd hammer`
+
+  `scons bindings=ruby`
+
+  `sudo scons bindings=ruby install`
+
+2. On linux, you will have to do 
+
+  `sudo ldconfig`
+  
+3. Build the gem
+  `gem build hammer-parser.gemspec`
+
+4. Install the gem
+  `gem install hammer-parser-x.x.x.gem`
+
+
+## Examples
+
+Add hammer to your Gemfile. 
+
+  `gem 'hammer-parser'`
+
+Use hammer in your project.
+
+  `require 'hammer-parser'`
+
+### Building a parser
+
+```ruby
+parser = Hammer::Parser.build {
+  token 'Hello '
+  choice {
+    token 'Mom'
+    token 'Dad'
+  }
+  token '!'
+}
+```
+
+Also possible:
+
+```ruby
+parser = Hammer::ParserBuilder.new
+  .token('Hello ')
+  .choice(Hammer::Parser.token('Mom'), Hammer::Parser.token('Dad'))
+  .token('!')
+  .build
+```
+
+More like hammer in C:
+
+```ruby
+h = Hammer::Parser
+parser = h.sequence(h.token('Hello '), h.choice(h.token('Mom'), h.token('Dad')), h.token('!'))
+```
+
+### Parsing
+
+```ruby
+result = parser.parse 'Hello Mom!'
+=> #<HParseResult>
+result = parser.parse 'Hello Someone!'
+=> nil
+```
+
+The `parse` method returns an `HParseResult` object, which needs to be
+kept around until you're entirely done with the parse tree, which can
+be accessed with `result.ast`.
+
+While the AST can be accessed using the same interface as the C
+HParsedToken type, we recommend using `result.ast.unmarshal` instead.
+This converts the entire parse tree into a standalone Ruby-native
+datastructure which will likely be much easier to work with.

data/lib/hammer-parser.rb

@@ -0,0 +1,47 @@
+require 'hammer/internal'
+require 'hammer/parser'
+require 'hammer/parser_builder'
+
+# Leave this in for now to be able to play around with HParseResult in irb.
+x = nil
+parser = Hammer::Parser.build {
+  token 'abc'
+  x = indirect
+  end_p
+}
+x.bind(Hammer::Parser.token('abd'))
+
+#$p = parser
+$r = parser.parse 'abcabd'
+
+#p $r[:ast][:data][:seq].elements.map {|e| e[:data][:bytes].token }
+
+
+h = Hammer::Parser
+parser =
+  h.many(
+    h.action(h.uint8) { |r|
+      #p "TT=#{r[:ast][:token_type]}, value=#{r[:ast][:data][:uint]}"
+      r.data * 2
+    })
+
+#parser = Hammer::Parser.build {
+#  many {
+#    uint8
+#    action { |r|
+#      p r
+#      r[:ast]
+#    }
+#  }
+#}
+
+$r = parser.parse 'abcdefgh'
+
+#p $r[:ast][:data][:seq].elements.map {|e| e[:data][:uint]}
+# or:
+#p $r.ast.data.map(&:data)
+
+
+h = Hammer::Parser
+parser = h.many(h.attr_bool(h.uint8) { |r| r.data <= 100 })
+#p parser.parse('abcdefgh').ast.data.map(&:data)

data/lib/hammer/internal.rb

@@ -0,0 +1,346 @@
+require 'ffi'
+
+module Hammer
+  module Internal
+    extend FFI::Library
+
+    ffi_lib 'hammer'
+
+    class DynamicVariable
+      SYMBOL_PREFIX = "Hammer::Internal::DynamicVariable gensym "
+      @@current_symbol = 0
+
+      def initialize(default=nil, name=nil, &block)
+        # This can take either a default value or a block.  If a
+        # default value is given, all threads' dynvars are initialized
+        # to that object.  If a block is given, the block is lazilly
+        # called on each thread to generate the initial value.  If
+        # both a block and a default value are passed, the block is
+        # called with the literal value.
+        @default = default
+        @block = block || Proc.new{|x| x}
+        @@current_symbol += 1
+        @sym = (SYMBOL_PREFIX + @@current_symbol.to_s).to_sym
+      end
+
+      def value
+        if Thread.current.key? @sym
+          return Thread.current[@sym]
+        else
+          return Thread.current[@sym] = @block.call(@default)
+        end
+      end
+
+      def value=(new_value)
+        Thread.current[@sym] = new_value
+      end
+
+      def with(new_value, &block)
+        old_value = value
+        begin
+          self.value = new_value
+          return block.call
+        ensure
+          self.value = old_value
+        end
+      end
+    end
+
+    # Maybe we can implement Hammer::Parser with FFI::DataConverter.
+    # That way, most hammer functions won't need to be wrapped.
+    # (Probably need to wrap token, sequence and choice only).
+    # See http://www.elabs.se/blog/61-advanced-topics-in-ruby-ffi
+    typedef :pointer, :h_parser
+
+    class HTokenType
+      extend FFI::DataConverter
+
+      @@known_type_map = {
+        :none => 1,
+        :bytes => 2,
+        :sint => 4,
+        :uint => 8,
+        :sequence => 16,
+      }
+
+      @@inverse_type_map = @@known_type_map.invert
+
+      @@from_hpt = {
+        :none => Proc.new { nil },
+        :bytes => Proc.new {|hpt| hpt[:data][:bytes].token},
+        :sint => Proc.new {|hpt| hpt[:data][:sint]},
+        :uint => Proc.new {|hpt| hpt[:data][:uint]},
+        :sequence => Proc.new {|hpt| hpt[:data][:seq].map {|x| x.unmarshal}},
+      }
+
+      def self.new(name, &block)
+        if name.is_a?(Symbol)
+          name_sym = name
+          name_str = name.to_s
+        else
+          name_str = name.to_s
+          name_sym = name.to_sym
+        end
+        num = Hammer::Internal.h_allocate_token_type(name_str)
+        @@known_type_map[name_sym] = num
+        @@inverse_type_map[num] = name_sym
+        @@from_hpt[name_sym] = block
+      end
+
+      def self.from_name(name)
+        unless @@known_type_map.key? name
+          num =  Hammer::Internal.h_get_token_type_number(name.to_s)
+          if num <= 0
+            raise ArgumentError, "Unknown token type #{name}"
+          end
+          @@known_type_map[name] = num
+          @@inverse_type_map[num] = name
+        end
+        return @@known_type_map[name]
+      end
+
+      def self.from_num(num)
+        unless @@inverse_type_map.key? num
+          name = Hammer::Internal.h_get_token_type_name(num)
+          if name.nil?
+            return nil
+          end
+          name = name.to_sym
+          @@known_type_map[name] = num
+          @@inverse_type_map[num] = name
+        end
+        return @@inverse_type_map[num]
+      end
+
+      def self.native_type
+        FFI::Type::INT
+      end
+
+      def self.to_native(val, ctx)
+        return val if val.is_a?(Integer)
+        return from_name(val)
+      end
+
+      def self.from_native(val, ctx)
+        return from_num(val) || val
+      end
+    end
+
+    # Define these as soon as possible, so that they can be used
+    # without fear elsewhere
+    attach_function :h_allocate_token_type, [:string], :int
+    attach_function :h_get_token_type_number, [:string], :int
+    attach_function :h_get_token_type_name, [:int], :string
+
+    class HCountedArray < FFI::Struct
+      layout  :capacity, :size_t,
+              :used, :size_t,
+              :arena, :pointer,
+              :elements, :pointer # HParsedToken**
+
+      def length
+        self[:used]
+      end
+
+      def elements
+        elem_array = FFI::Pointer.new(:pointer, self[:elements])
+        return (0...self[:used]).map { |i| HParsedToken.new(elem_array[i].read_pointer) }
+      end
+
+      #def [](idx)
+      #  raise ArgumentError, "Index out of range" unless idx >= 0 and idx < length
+      #  elem_array = FFI::Pointer.new(:pointer, self[:elements])
+      #  return HParsedToken.new(elem_array[i].read_pointer)
+      #end
+
+      def map(&code)
+        elements.map {|x| code.call x}
+      end
+      def each(&code)
+        elements.each {|x| code.call x}
+      end
+    end
+
+    class HBytes < FFI::Struct
+      layout  :token, :pointer, # uint8_t*
+              :len, :size_t
+
+      def token
+        # TODO: Encoding?
+        # Should be the same encoding as the string the token was created with.
+        # But how do we get to this knowledge at this point?
+        # Cheap solution: Just ask the user (additional parameter with default value of UTF-8).
+        self[:token].read_string(self[:len])
+      end
+
+      # TODO: Probably should rename this to match ruby conventions: length, count, size
+      def len
+        self[:len]
+      end
+    end
+
+    class HString < FFI::Struct
+      layout :content, HBytes.by_ref,
+             :encoding, :uint64
+      def token
+        return self[:content].token.force_encoding(
+                  ObjectSpace._id2ref(self[:encoding]))
+      end
+    end
+
+    HTokenType.new(:"com.upstandinghackers.hammer.ruby.encodedStr") {|hpt|
+        hpt.user(HString).token
+      }
+    HTokenType.new(:"com.upstandinghackers.hammer.ruby.object") {|hpt|
+        ObjectSpace._id2ref(hpt[:data][:uint])
+      }
+
+    class HParsedTokenDataUnion < FFI::Union
+      layout  :bytes, HBytes.by_value,
+              :sint, :int64,
+              :uint, :uint64,
+              :dbl, :double,
+              :flt, :float,
+              :seq, HCountedArray.by_ref,
+              :user, :pointer
+    end
+
+    class HParsedToken < FFI::Struct
+      layout  :token_type, HTokenType,
+              :data, HParsedTokenDataUnion.by_value,
+              :index, :size_t,
+              :bit_offset, :char
+
+      def normalize
+        # If I'm null, return nil.
+        return nil if null?
+        return self
+      end
+      
+      def token_type
+        self[:token_type]
+      end
+
+      # TODO: Is this name ok?
+      def data
+        return self[:data][:bytes].token if token_type == :bytes
+        return self[:data][:sint] if token_type == :sint
+        return self[:data][:uint] if token_type == :uint
+        return self[:data][:seq].elements if token_type == :sequence
+        return self[:data][:user] if token_type == :user
+      end
+
+      def bytes
+        raise ArgumentError, 'wrong token type' unless token_type == :bytes
+        self[:data][:bytes]
+      end
+
+      def seq
+        raise ArgumentError, 'wrong token type' unless token_type == :sequence
+        self[:data][:seq]
+      end
+
+      def index
+        self[:index]
+      end
+
+      def bit_offset
+        self[:bit_offset]
+      end
+
+      def user(struct)
+        struct.by_ref.from_native(self[:data][:user], nil)
+      end
+
+      def unmarshal
+        Hammer::Internal::HTokenType.class_variable_get(:@@from_hpt)[token_type].call self
+      end
+    end
+
+    class HParseResult < FFI::Struct
+      layout  :ast, HParsedToken.by_ref,
+              :bit_length, :long_long,
+              :arena, :pointer
+
+      def ast
+        self[:ast].normalize
+      end
+
+      def bit_length
+        self[:bit_length]
+      end
+
+      def self.release(ptr)
+        Hammer::Internal.h_parse_result_free(ptr) unless ptr.null?
+      end
+
+      def arena_alloc(type)
+        Hammer::Internal.arena_alloc(self[:arena], type)
+      end
+    end
+
+    def self.arena_alloc(arena, type)
+      ptr = h_arena_malloc(arena, type.size)
+      return type.by_ref.from_native(ptr, nil)
+    end
+
+    # run a parser
+    attach_function :h_parse, [:h_parser, :pointer, :size_t], HParseResult.auto_ptr # TODO: Use :buffer_in instead of :string?
+
+    # build a parser
+    attach_function :h_token, [:buffer_in, :size_t], :h_parser
+    attach_function :h_ch, [:uint8], :h_parser
+    attach_function :h_ch_range, [:uint8, :uint8], :h_parser
+    attach_function :h_int_range, [:h_parser, :int64, :int64], :h_parser
+    attach_function :h_bits, [:size_t, :bool], :h_parser
+    attach_function :h_int64, [], :h_parser
+    attach_function :h_int32, [], :h_parser
+    attach_function :h_int16, [], :h_parser
+    attach_function :h_int8,  [], :h_parser
+    attach_function :h_uint64, [], :h_parser
+    attach_function :h_uint32, [], :h_parser
+    attach_function :h_uint16, [], :h_parser
+    attach_function :h_uint8,  [], :h_parser
+    attach_function :h_whitespace, [:h_parser], :h_parser
+    attach_function :h_left,   [:h_parser, :h_parser], :h_parser
+    attach_function :h_right,  [:h_parser, :h_parser], :h_parser
+    attach_function :h_middle, [:h_parser, :h_parser, :h_parser], :h_parser
+    attach_function :h_in, [:pointer, :size_t], :h_parser
+    attach_function :h_not_in, [:pointer, :size_t], :h_parser
+    attach_function :h_end_p, [], :h_parser
+    attach_function :h_nothing_p, [], :h_parser
+    attach_function :h_sequence, [:varargs], :h_parser
+    attach_function :h_choice, [:varargs], :h_parser
+    attach_function :h_butnot, [:h_parser, :h_parser], :h_parser
+    attach_function :h_difference, [:h_parser, :h_parser], :h_parser
+    attach_function :h_xor, [:h_parser, :h_parser], :h_parser
+    attach_function :h_many, [:h_parser], :h_parser
+    attach_function :h_many1, [:h_parser], :h_parser
+    attach_function :h_repeat_n, [:h_parser, :size_t], :h_parser
+    attach_function :h_optional, [:h_parser], :h_parser
+    attach_function :h_ignore, [:h_parser], :h_parser
+    attach_function :h_sepBy, [:h_parser, :h_parser], :h_parser
+    attach_function :h_sepBy1, [:h_parser, :h_parser], :h_parser
+    attach_function :h_epsilon_p, [], :h_parser
+    attach_function :h_length_value, [:h_parser, :h_parser], :h_parser
+    attach_function :h_and, [:h_parser], :h_parser
+    attach_function :h_not, [:h_parser], :h_parser
+
+    attach_function :h_indirect, [], :h_parser
+    attach_function :h_bind_indirect, [:h_parser, :h_parser], :void
+
+    callback :HAction, [HParseResult.by_ref], HParsedToken.by_ref
+    attach_function :h_action, [:h_parser, :HAction], :h_parser
+
+    callback :HPredicate, [HParseResult.by_ref], :bool
+    attach_function :h_attr_bool, [:h_parser, :HPredicate], :h_parser
+
+    # free the parse result
+    attach_function :h_parse_result_free, [HParseResult.by_ref], :void
+
+    # TODO: Does the HParser* need to be freed?
+
+    # Add the arena
+    attach_function :h_arena_malloc, [:pointer, :size_t], :pointer
+  end
+end