abnf-parsing 0.2.0

checksums.yaml

@@ -0,0 +1,15 @@
+---
+!binary "U0hBMQ==":
+  metadata.gz: !binary |-
+    Y2Y2ZTliMTljZGZlYTA3YTJiOWUwMjJkM2FiOWFjM2NmYTFhNDY2OQ==
+  data.tar.gz: !binary |-
+    NjM4MzZjYjlkOTNkMWE2MWJjMTIxMDE4ODM3MmFjMTAyNzFhNGZkNA==
+SHA512:
+  metadata.gz: !binary |-
+    NTJhMWUxZmI4YzBmNGI4MWQzZjc3NjhlODI3NDVjOWE3YmM2NTA3NDBkYzdh
+    OWQ5MDQzNzI1ZDY3ZjQ1ZTcyNGUyZjYzN2ZhOWU5N2U1ZTFiYmNjYmU4ZTE3
+    NjQzODdhZjQ2ZmJhOTk4NzJmZTdlOWViNjRhMjc4OWY2OTgxOTQ=
+  data.tar.gz: !binary |-
+    NzAzMzg5MjBlMzk4ODIwYmFlN2JkYzJkMDI5ZTAyYTMxYWNlOWRhODg2MzUy
+    MjQwMjZjYzEyNTI5OTA2MzYyZGVkZjg0NGFjMzA1MzA5YTFmZDUyMTA4MjIx
+    MTZmY2YwMDQ5ZjYwODgwNGJlNDc1MWNiMTA5MzhlMTkzNjYzYjE=

data/README

@@ -0,0 +1,25 @@
+
+== Overview ==
+
+This is a simple library for easily implementing parsers that are specified in Augmented Backus-Naur Form (ABNF).
+Many IETF RFCs specify protocols and other entities using ABNF. This library lets you use those core operators and
+rules almost directly as Ruby objects. An optional block can be specified for a rule. If the rule matches, it will
+pass the match into the block. This makes it possible to build up state as a parse progresses.
+
+This version of the library was written to RFC 2234, which is an obsoleted version of the RFC. Not too much
+changed in the subsequent versions however. I'm hoping to do a pass through soon and make sure this library
+is compliant with RFC 5234.
+
+Check out: http://tools.ietf.org/html/rfc5234 for more information.
+
+== Example ==
+
+For a detailed example, tests/url.rb contains a full URL parser using the ABNF grammar from RFC 3986. It parses
+a URL from a string returns a structure containing the components.
+
+== Caveats ==
+
+* There's no lookahead, so when using Alternate, specify your longest matches first if one of the smaller alternative
+matches could potentially fire.
+
+* It works on an 8-bit character basis - at least for now.

data/Rakefile

@@ -0,0 +1,40 @@
+
+
+
+
+require 'rubygems'
+require 'rake'
+require 'rake/testtask'
+require 'rubygems/package_task'
+
+Rake::TestTask.new do |t|
+  t.libs << "test"
+  t.test_files = FileList['test/*.rb']
+  t.verbose = true
+end
+
+GEM = "abnf-parsing"
+
+spec = Gem::Specification.new do |s|
+  s.name = GEM
+  s.version = "0.2.0"
+  s.platform = Gem::Platform::RUBY
+  s.has_rdoc = true
+  s.extra_rdoc_files = ["README", "TODO"]
+  s.summary = "ABNF parsing in Ruby"
+  s.description = "A Ruby library for implementing parsers specified with Augmented Backus Naur Form (ABNF)."
+  s.authors = ["Rob Day", "Ryan Tecco"]
+  s.homepage = "https://github.com/rkday/ruby-abnf"
+  s.email = "ruby-abnf@rkd.me.uk"
+  s.require_path = "lib"
+  s.files = %w{README Rakefile TODO} + Dir.glob("{lib,test}/*")
+end
+
+Gem::PackageTask.new(spec) do |pkg|
+
+end
+
+desc "Create the gemspec file."
+task :gemspec do
+  File.open("#{GEM}.gemspec", "w"){|f| f.puts spec.to_ruby}
+end

data/TODO

@@ -0,0 +1,4 @@
+* ABNF in ABNF. Write an ABNF grammar to parse and create parser objects from an ABNF spec.
+* Test query parsing (with another ABNF parser).
+* On match, call a method (if defined) (or pass to a block like currently (if defined)).
+* Update to RFC 5234?

data/lib/abnf.rb

@@ -0,0 +1,264 @@
+
+# ABNF (RFC 2234) parser objects.
+
+module ABNF
+  class ::Range
+    def until_end # &blk
+      self.last.times{|i| yield i}
+    end
+  end
+
+  class RangeWithInfiniteUpperBound
+    def initialize(first)
+      @first = first
+    end
+
+    def first
+      @first
+    end
+
+    def until_end # &blk
+      i = 0
+
+      while true
+        yield i
+        i += 1
+      end
+    end
+  end
+
+  class Stream
+    attr_reader :pos
+
+    def initialize(s, full = nil, pos = 0)
+      @s = s
+      @full = full.nil? ? s : full
+      @pos = pos
+    end
+
+    def first
+      @s[0].ord if @s[0]
+    end
+
+    def rest
+      Stream.new(@s[1..-1], @full, @pos + 1)
+    end
+
+    def clip(start)
+      @full[start...@pos]
+    end
+  end
+
+  #
+  # Helper Classes
+
+  class Satisfies
+
+    # Takes a stream, returns a stream or failure.
+    def match(s)
+      s.rest if predicate(s.first)
+    end
+  end
+
+  class Range < Satisfies
+    def initialize(r)
+      @r = r
+    end
+
+    def predicate(c)
+      @r.member?(c)
+    end
+  end
+
+  #
+  # Core Operators
+
+  # Remember to place the longest matches first!
+  class Alternate
+    def initialize(*choices, &blk)
+      @choices = choices
+      @blk = blk
+    end
+
+    def set_block(&blk)
+      @blk = blk
+      return self
+    end
+
+    def match(strm)
+      start = strm.pos
+
+      @choices.each {
+        |c|
+
+        n_strm = c.match(strm);
+
+        if n_strm
+          @blk.call(n_strm.clip(start)) unless @blk.nil?
+
+          return n_strm
+        end
+      }
+
+      return nil
+    end
+  end
+
+  class AlternateChars < Alternate
+    def initialize(string, &blk)
+      @choices = string.each_char.map { |c| Char.new(c) }
+      @blk = blk
+    end
+  end
+
+  class Concat
+    def initialize(*choices, &blk)
+      @choices = choices
+      @blk = blk
+    end
+
+    def set_block(&blk)
+      @blk = blk
+      return self
+    end
+
+    def match(strm)
+      c_strm = strm
+      start = c_strm.pos
+
+      @choices.each {
+        |c|
+        
+        c_strm = c.match(c_strm)
+
+        return nil if c_strm.nil?
+      }
+
+      @blk.call(c_strm.clip(start)) unless @blk.nil?
+
+      return c_strm
+    end
+  end
+
+  class Literal < Concat
+    def initialize(string, &blk)
+      @choices = string.each_char.map { |c| Char.new(c) }
+      @blk = blk
+    end
+  end
+
+  class Repetition
+    # Spec: range (between), integer (exact), [:at_most, N], [:at_least, N], :any (zero or more)
+    def initialize(spec, what, &blk)
+      @spec = spec
+      @what = what
+      @blk = blk
+    end
+
+    def set_block(&blk)
+      @blk = blk
+      return self
+    end
+
+    def match(strm)
+      c_strm = strm
+      start = strm.pos
+
+      r = \
+        case @spec
+        when Array # :at_least, :at_most
+          option, i = @spec
+
+          if option == :at_most
+            (0..i)
+          elsif option == :at_least
+            RangeWithInfiniteUpperBound.new(i)
+          end
+        when Integer # Exact
+          @spec..@spec
+        when ::Range # Between
+          @spec
+        when Symbol # Any (zero or more)
+          RangeWithInfiniteUpperBound.new(0)
+        end
+
+      r.until_end {
+        |i|
+
+        tried = @what.match(c_strm)
+
+        if tried.nil?
+          if i < r.first
+            return nil
+          else
+            break
+          end
+        else
+          c_strm = tried
+        end
+      }
+
+      @blk.call(c_strm.clip(start)) unless @blk.nil?
+
+      c_strm
+    end
+  end
+
+  class Optional < Repetition
+    def initialize(what)
+      super([:at_most, 1], what)
+    end
+  end
+
+  class OptionalConcat < Optional
+    def initialize(*what)
+      super(Concat.new(*what))
+    end
+  end
+
+  #
+  # Core Rules
+  #
+  # I tried to preserve the RFC names where possible.
+
+  class Char < Satisfies
+    def initialize(c)
+      @char = c.ord
+    end
+
+    def predicate(c)
+      c == @char
+    end
+  end
+
+  class Alpha < Alternate; def initialize; super(Range.new(0x41..0x5A), Range.new(0x61..0x7A)) end end
+  class AsciiChar < Range; def initialize; super(0x1..0x7F) end end # char
+  class Bit < Alternate; def initialize; super(Char.new(?0), Char.new(?1)) end end
+  class CR < Char; def initialize; super(0x0D) end end # carriage return
+  class LF < Char; def initialize; super(0x0A) end end # line feed
+  class CRLF < Concat; def initialize; super(CR.new, LF.new) end end # Internet standard newline
+  class Ctl < Alternate; def initialize; super(Range.new(0..0x1F), Char.new(0x7F)) end end # control characters
+  class Digit < Range; def initialize; super(0x30..0x39) end end
+  class DQuote < Char; def initialize; super(0x22) end end # double quote
+  class HTab < Char; def initialize; super(0x9) end end # horizontal tab
+  class SP < Char; def initialize; super(0x20) end end # space
+  class Octet < Range; def initialize; super(0..255) end end # any 8-bit data value
+  class VChar < Range; def initialize; super(0x21..0x7E) end end # visible (printing) characters
+  class WSP < Alternate; def initialize; super(SP.new, HTab.new) end end # whitespace
+
+  class HexDigit < Alternate
+    def initialize
+      super(Digit.new, Range.new(0x41..0x46), Range.new(0x61..0x66))
+    end
+  end
+
+  class LWSP < Repetition # Linear white space (past newline)
+    def initialize
+      super(:any, Alternate.new(WSP.new, Concat.new(CRLF.new, WSP.new)))
+    end
+  end
+
+  def parse(parser, str)
+    parser.match(Stream.new(str))
+  end
+end

data/test/basics.rb

@@ -0,0 +1,123 @@
+$: << File.join(File.dirname(__FILE__), "../lib")
+
+require 'test/unit'
+require 'abnf'
+
+class ABNFTest < Test::Unit::TestCase
+  include ABNF
+
+  def assert_blank(s)
+    assert s.nil? || (s.class == String && s.empty?)
+  end
+
+  def test_variable_repetition
+    assert parse(Repetition.new(3, Alpha.new), "abcdefg")
+    assert parse(Repetition.new(1, Alpha.new), "a12345")
+    assert_nil parse(Repetition.new(3, Alpha.new), "ab123")
+    assert_nil parse(Repetition.new(1, Alpha.new), "123456")
+    assert parse(Repetition.new(1..3, Alpha.new), "ab1234")
+    assert_nil parse(Repetition.new(3..5, Alpha.new), "ab1234")
+    assert parse(Repetition.new(0, Alpha.new), "12345")
+    assert parse(Repetition.new(0..4, Alpha.new), "a1234")
+    assert parse(Repetition.new(0..1, Alpha.new), "1234")
+    assert parse(Repetition.new(0..1, Alpha.new), "a1234")
+    assert parse(Repetition.new(0..1, Alpha.new), "ab1234")
+    assert parse(Repetition.new([:at_least, 2], Alpha.new){|r| assert_equal "ab", r}, "ab1234")
+    assert parse(Repetition.new([:at_least, 2], Alpha.new){|r| assert_equal "abcde", r}, "abcde12")
+    assert parse(Repetition.new([:at_most, 3], Alpha.new){|r| assert_equal "abc", r}, "abcdefgh")
+    assert parse(Repetition.new([:at_most, 3], Alpha.new), "12345")
+    assert parse(Repetition.new([:at_most, 3], Alpha.new){|r| assert_equal "ab", r}, "ab12345")
+    assert parse(Concat.new(Optional.new(Digit.new), Alpha.new), "a")
+    assert parse(Concat.new(Optional.new(Digit.new), Alpha.new), "1a")
+    assert parse(Repetition.new(:any, Alpha.new), "12345")
+    assert parse(Repetition.new(:any, Alpha.new), "abcdefg12345")
+  end
+
+  def test_single_char_parse
+    parser = Char.new(?-)
+    assert_nil parse(parser, ")")
+    assert parse(parser, "-")
+  end
+
+  def test_alternate_char_parse
+    parser = AlternateChars.new("abcdef")
+    assert_nil parse(parser, ")")
+    assert_nil parse(parser, "-")
+    assert parse(parser, "a")
+    assert parse(parser, "b")
+    assert parse(parser, "c")
+    assert parse(parser, "d")
+    assert parse(parser, "e")
+    assert parse(parser, "f")
+    assert_nil parse(parser, "g")
+  end
+
+  def test_literal_parse
+    parser = Literal.new("abcdef")
+    
+    assert parse(parser, "abcdef")
+    
+    assert_nil parse(parser, ")")
+    assert_nil parse(parser, "-")
+    assert_nil parse(parser, "a")
+    assert_nil parse(parser, "abcd")
+  end
+
+  def test_set_block
+    output = nil
+    parser = Literal.new("abcdef")
+    
+    assert parse(parser, "abcdef")
+    assert_nil output
+    
+    parser.set_block {|m| output = m}
+    
+    assert parse(parser, "abcdef")
+    
+    assert (output == "abcdef")
+  end
+
+  def test_optional_concat_parse
+    # Parser represents ["a" "b"] "c"
+    parser = Concat.new(OptionalConcat.new(Char.new(?a), Char.new(?b)), Char.new(?c))
+    
+    assert parse(parser, "abc")
+    assert parse(parser, "c")
+    
+    assert_nil parse(parser, ")")
+    assert_nil parse(parser, "-")
+    assert_nil parse(parser, "a")
+    assert_nil parse(parser, "ab")
+    assert_nil parse(parser, "ac")
+  end
+
+  def test_example_1
+
+    # Phone Number
+    p1 = Concat.new(Repetition.new(3, Digit.new), Char.new(?-), Repetition.new(4, Digit.new))
+    assert_nil parse(p1, "22-3452")
+    assert_nil parse(p1, "hello world")
+    assert_nil parse(p1, "4293-603")
+    assert_nil parse(p1, "")
+    assert_nil parse(p1, "4293-3603")
+    assert_nil parse(p1, "734-904-2840")
+    assert parse(p1, "429-3603")
+
+    # Phone Number w/ optional Area Code, same delimiters
+    phone_number = Concat.new(Repetition.new(3, Digit.new), Char.new(?-), Repetition.new(4, Digit.new)){|pn| assert_equal "904-2840", pn}
+    area_code = Concat.new(Repetition.new(3, Digit.new), Char.new(?-))
+    p2 = Alternate.new(Concat.new(area_code, phone_number){|f| assert_equal "734-904-2840", f}, phone_number)
+    assert_nil parse(p2, "22-3452")
+    assert_nil parse(p2, "hello world")
+    assert_nil parse(p2, "4293-603")
+    assert_nil parse(p2, "")
+    assert_nil parse(p2, "4293-3603")
+    assert parse(p2, "904-2840")
+    assert parse(p2, "734-904-2840")
+
+    # Phone Number w/ optional Area Code, different delimiters
+    area_code = Concat.new(Char.new(?(), Repetition.new(3, Digit.new), Char.new(?)))
+    p3 = Concat.new(Optional.new(area_code), phone_number)
+    assert parse(p3, "904-2840")
+  end
+end

data/test/url.rb

@@ -0,0 +1,206 @@
+$: << File.join(File.dirname(__FILE__), "../lib")
+
+require 'test/unit'
+require 'abnf'
+
+# The rules and state for a URL parser. Taken almost verbatim from RFC 3986.
+# There is no support for IP-literal, IPvFuture, or IPv6address.
+# These rules are not implemented: absolute_uri, path, reserved and gen_delims although there is
+# no functional reason that they couldn't be.
+
+class URLParser  
+  include ABNF
+
+  URL = Struct.new("URL", :fragment, :host, :path, :port, :scheme, :user)
+  
+  # API
+  
+  def parse(str)
+    @result = Struct::URL.new
+    
+    if uri_reference.match(Stream.new(str))
+      @result
+    else
+      raise RuntimeError.new("Parse error")
+    end
+  end
+  
+  # Rules
+  
+  def scheme
+    Concat.new(Alpha.new, Repetition.new(:any, Alternate.new(Alpha.new, Digit.new,
+      Char.new(?+), Char.new(?-), Char.new(?.)))){|s| @result.scheme = s}
+  end
+  
+  def dec_octet
+    Alternate.new(Concat.new(Char.new(?2), Char.new(?5), Range.new(0x30..0x35)), # 250-255    
+      Concat.new(Char.new(?2), Range.new(0x30..0x34), Digit.new), # 200-249
+      Concat.new(Char.new(?1), Repetition.new(2, Digit.new)), # 100-199
+      Concat.new(Range.new(0x31..0x39), Digit.new), # 10-99                  
+      Digit.new) # 0-9
+  end
+  
+  def pct_encoded
+    Concat.new(Char.new(?%), HexDigit.new, HexDigit.new)
+  end
+  
+  def sub_delims
+    Alternate.new(Char.new(?!), Char.new(?$), Char.new(?&), Char.new(?'), Char.new(?(), 
+      Char.new(?)), Char.new(?*), Char.new(?+), Char.new(?,), Char.new(?;), Char.new(?=))
+  end
+  
+  def unreserved
+    Alternate.new(Alpha.new, Digit.new, Char.new(?-), Char.new(?.), Char.new(?_), Char.new(?~))
+  end
+  
+  def pchar
+    Alternate.new(unreserved, pct_encoded, sub_delims, Char.new(?:), Char.new(?@))
+  end
+  
+  def ipv4_address
+    Concat.new(dec_octet, Char.new(?.), dec_octet, Char.new(?.), dec_octet, Char.new(?.), dec_octet)
+  end
+  
+  def reg_name
+    Repetition.new(:any, Alternate.new(unreserved, pct_encoded, sub_delims))
+  end
+  
+  def host
+    Alternate.new(ipv4_address, reg_name){|host| @result.host = host}
+  end
+  
+  def port
+    Repetition.new(:any, Digit.new){|port| @result.port = port.to_i}
+  end
+  
+  def userinfo
+    Repetition.new(:any, Alternate.new(unreserved, pct_encoded, sub_delims, Char.new(?:))){|u| @result.user = u}
+  end
+  
+  def authority
+    Concat.new(Optional.new(Concat.new(userinfo, Char.new(?@))), host, Optional.new(Concat.new(Char.new(?:), port)))
+  end
+    
+  def segment
+    Repetition.new(:any, pchar)
+  end
+
+  def segment_nz
+    Repetition.new([:at_least, 1], pchar)
+  end
+  
+  def segment_nz_nc # non-zero-length segment without any colon
+    Repetition.new([:at_least, 1], Alternate.new(unreserved, pct_encoded, sub_delims, Char.new(?@)))
+  end
+      	  
+  def path_abempty
+    Repetition.new(:any, Concat.new(Char.new(?/), segment)){|path| @result.path = path}
+  end
+  
+  def path_absolute
+    Concat.new(Char.new(?/),
+      Optional.new(Concat.new(segment_nz, Repetition.new(:any, Concat.new(Char.new(?/), segment))))){|path| @result.path = path}
+  end
+
+  def path_empty
+    Repetition.new(0, pchar){|path| @result.path = path}
+  end
+
+  def path_noscheme
+    Concat.new(segment_nz_nc, Repetition.new(:any, Concat.new(Char.new(?/), segment)))
+  end
+  
+  def path_rootless
+    Concat.new(segment_nz, Repetition.new(:any, Concat.new(Char.new(?/), segment))){|path| @result.path = path}
+  end
+    	
+  def hier_part
+    Alternate.new(Concat.new(Char.new(?/), Char.new(?/), authority, path_abempty),
+      path_absolute,
+      path_rootless,
+      path_empty)
+  end
+  
+  def query
+    Repetition.new(:any, Alternate.new(pchar, Char.new(?/), Char.new(??))){|query| puts query}
+  end
+  
+  def fragment
+    Repetition.new(:any, Alternate.new(pchar, Char.new(?/), Char.new(??))){|fragment| @result.fragment = fragment}
+  end
+  
+  def uri
+    Concat.new(scheme,
+      Char.new(?:),
+      hier_part,
+      Optional.new(Concat.new(Char.new(??), query)),
+      Optional.new(Concat.new(Char.new(?#), fragment)))
+  end
+
+  def relative_part
+    Alternate.new(Concat.new(Char.new(?/), Char.new(?/), authority, path_abempty),
+      path_absolute,
+      path_noscheme,
+      path_empty)
+  end
+  
+  def relative_ref
+    Concat.new(relative_part,
+      Optional.new(Concat.new(Char.new(??), query)),
+      Optional.new(Concat.new(Char.new(?#), fragment)))
+  end
+  
+  def uri_reference
+    Alternate.new(uri, relative_ref)
+  end
+end
+
+class ABNFTest < Test::Unit::TestCase
+  include ABNF
+  
+  def test_url_parser
+        
+    #
+    # URLs
+        
+    u = URLParser.new.parse("http://karmalab.org")
+    assert u
+    assert_equal "http", u.scheme
+    assert_equal "karmalab.org", u.host
+    assert_nil u.port
+    assert_equal "", u.path
+    
+    u = URLParser.new.parse("http://rt@karmalab.org:9001")
+    assert u
+    assert_equal "http", u.scheme
+    assert_equal "karmalab.org", u.host
+    assert_equal 9001, u.port
+    assert_equal "rt", u.user
+    assert_equal "", u.path
+    
+    u = URLParser.new.parse("file:///tmp/file.txt")
+    assert u
+    assert_equal "file", u.scheme
+    assert_equal "/tmp/file.txt", u.path
+    assert_blank u.host
+    assert_blank u.port
+    assert_blank u.user
+    
+    u = URLParser.new.parse("http://triggit.com/j?u=rt10880&p=http://lambda/#fragment16")
+    assert u
+    assert_equal "http", u.scheme
+    assert_equal "triggit.com", u.host
+    assert_equal "/j", u.path
+    assert_equal "fragment16", u.fragment
+    
+    u = URLParser.new.parse("about:config")
+    assert u
+    assert_equal "about", u.scheme
+    assert_equal "config", u.path
+    
+    u = URLParser.new.parse("/path/to/file.html#fr23-1")
+    assert u
+    assert_equal "/path/to/file.html", u.path
+    assert_equal "fr23-1", u.fragment
+  end
+end

metadata

@@ -0,0 +1,52 @@
+--- !ruby/object:Gem::Specification
+name: abnf-parsing
+version: !ruby/object:Gem::Version
+  version: 0.2.0
+platform: ruby
+authors:
+- Rob Day
+- Ryan Tecco
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2014-03-09 00:00:00.000000000 Z
+dependencies: []
+description: A Ruby library for implementing parsers specified with Augmented Backus
+  Naur Form (ABNF).
+email: ruby-abnf@rkd.me.uk
+executables: []
+extensions: []
+extra_rdoc_files:
+- README
+- TODO
+files:
+- README
+- Rakefile
+- TODO
+- lib/abnf.rb
+- test/basics.rb
+- test/url.rb
+homepage: https://github.com/rkday/ruby-abnf
+licenses: []
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.2.2
+signing_key: 
+specification_version: 4
+summary: ABNF parsing in Ruby
+test_files: []