abnf 0.0.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: dd62c9239754c0264cfc03bd29bd5a3f5473f9d2
+  data.tar.gz: 68adb3f6b144396412d52379021fa5d0c32b7676
+SHA512:
+  metadata.gz: c8a9a996d80087aa76a05fb19e2fb94fa736e19549787cadb12c31e10f358175ff9640856bfafe7e5b0de85df0aae079036332719e4161c8d89ad51e651b15ae
+  data.tar.gz: 379e25d8d244b2f2154a8e934943a42084bd05b70d301533eb82ba85587111b30eda0e01b7c762345d1292e142bdab9ba409a3b4a0a1993eb49218ece2c7aa4a

data/.gitignore

@@ -0,0 +1,17 @@
+*.gem
+*.rbc
+.bundle
+.config
+.yardoc
+Gemfile.lock
+InstalledFiles
+_yardoc
+coverage
+doc/
+lib/bundler/man
+pkg
+rdoc
+spec/reports
+test/tmp
+test/version_tmp
+tmp

data/.travis.yml

@@ -0,0 +1,16 @@
+language: ruby
+rvm:
+  - 1.8.7
+  - 1.9.2
+  - 1.9.3
+  - 2.0.0
+  - jruby-18mode
+  - jruby-19mode
+  - rbx-18mode
+  - rbx-19mode
+matrix:
+  allow_failures:
+    - rvm: jruby-18mode
+    - rvm: jruby-19mode
+    - rvm: rbx-18mode
+    - rvm: rbx-19mode

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in abnf.gemspec
+gemspec

data/LICENSE.txt

@@ -0,0 +1,22 @@
+Copyright (c) 2013 Steve Klabnik
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,194 @@
+# Abnf
+
+[![Build Status](https://travis-ci.org/steveklabnik/abnf.png?branch=master)](https://travis-ci.org/steveklabnik/abnf) [![Code Climate](https://codeclimate.com/github/steveklabnik/abnf.png)](https://codeclimate.com/github/steveklabnik/abnf)
+
+This is a library to convert ABNF (Augmented Backus-Naur Form) to Regexp
+(Regular Expression) written in Ruby.  It parses a description according to
+ABNF defined by RFC2234 and some variants.  Then the parsed grammar is
+transformed to recursion free.  Although the transformation is impossible in
+general, the library transform left- and right-recursion.  The recursion free
+grammar can be printed as Regexp literal which is suitable in Ruby script.  The
+literal is pretty readable because parentheses are minimized and properly
+indented by the Wadler's pretty printing algebra.  It is also possible to use
+the Regexp just in place.
+
+I (@steveklabnik) have gem-ified the code originally located at
+[https://github.com/akr/abnf](https://github.com/akr/abnf).
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+    gem 'abnf'
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install abnf
+
+## Usage
+
+Following example shows URI-reference defined by RFC 2396 can be converted to
+Regexp.
+Note that URI-reference has no recursion.
+Also note that the example works well without installing the library after make.
+
+  require 'pp'
+  require 'abnf'
+
+  # URI-reference [RFC2396]
+  pp ABNF.regexp_tree(<<-'End')
+        URI-reference = [ absoluteURI | relativeURI ] [ "#" fragment ]
+        absoluteURI   = scheme ":" ( hier_part | opaque_part )
+        relativeURI   = ( net_path | abs_path | rel_path ) [ "?" query ]
+
+        (...snip...)
+
+        lowalpha = "a" | "b" | "c" | "d" | "e" | "f" | "g" | "h" | "i" |
+                   "j" | "k" | "l" | "m" | "n" | "o" | "p" | "q" | "r" |
+                   "s" | "t" | "u" | "v" | "w" | "x" | "y" | "z"
+        upalpha  = "A" | "B" | "C" | "D" | "E" | "F" | "G" | "H" | "I" |
+                   "J" | "K" | "L" | "M" | "N" | "O" | "P" | "Q" | "R" |
+                   "S" | "T" | "U" | "V" | "W" | "X" | "Y" | "Z"
+        digit    = "0" | "1" | "2" | "3" | "4" | "5" | "6" | "7" |
+                   "8" | "9"
+  End
+
+The result is follows:
+  %r{(?:[a-z][\x2b\x2d\x2e0-9a-z]*:
+        (?:(?://
+              (?:(?:(?:(?:[!'-\x2a\x2d\x2e0-9_a-z~]|
+                          %[0-9a-f][0-9a-f]|
+                          [\x24&\x2b,:;=])*@)?
+                    (?:(?:(?:[0-9a-z]|[0-9a-z][\x2d0-9a-z]*[0-9a-z])\x2e)*
+                       (?:[a-z]|[a-z][\x2d0-9a-z]*[0-9a-z])\x2e?|
+                       \d+\x2e\d+\x2e\d+\x2e\d+)(?::\d*)?)?|
+                 (?:[!'-\x2a\x2d\x2e0-9_a-z~]|
+                    %[0-9a-f][0-9a-f]|
+                    [\x24&\x2b,:;=@])+)
+              (?:/
+                 (?:[!'-\x2a\x2d\x2e0-9_a-z~]|%[0-9a-f][0-9a-f]|[\x24&\x2b,:=@])*
+                 (?:;
+                    (?:[!'-\x2a\x2d\x2e0-9_a-z~]|
+                       %[0-9a-f][0-9a-f]|
+                       [\x24&\x2b,:=@])*)*
+                 (?:/
+                    (?:[!'-\x2a\x2d\x2e0-9_a-z~]|
+                       %[0-9a-f][0-9a-f]|
+                       [\x24&\x2b,:=@])*
+                    (?:;
+                       (?:[!'-\x2a\x2d\x2e0-9_a-z~]|
+                          %[0-9a-f][0-9a-f]|
+                          [\x24&\x2b,:=@])*)*)*)?|
+              /(?:[!'-\x2a\x2d\x2e0-9_a-z~]|%[0-9a-f][0-9a-f]|[\x24&\x2b,:=@])*
+              (?:;
+                 (?:[!'-\x2a\x2d\x2e0-9_a-z~]|
+                    %[0-9a-f][0-9a-f]|
+                    [\x24&\x2b,:=@])*)*
+              (?:/
+                 (?:[!'-\x2a\x2d\x2e0-9_a-z~]|%[0-9a-f][0-9a-f]|[\x24&\x2b,:=@])*
+                 (?:;
+                    (?:[!'-\x2a\x2d\x2e0-9_a-z~]|
+                       %[0-9a-f][0-9a-f]|
+                       [\x24&\x2b,:=@])*)*)*)
+           (?:\x3f(?:[!\x24&-;=\x3f@_a-z~]|%[0-9a-f][0-9a-f])*)?|
+           (?:[!'-\x2a\x2d\x2e0-9_a-z~]|%[0-9a-f][0-9a-f]|[\x24&\x2b,:;=\x3f@])
+           (?:[!\x24&-;=\x3f@_a-z~]|%[0-9a-f][0-9a-f])*)|
+        (?://
+           (?:(?:(?:(?:[!'-\x2a\x2d\x2e0-9_a-z~]|
+                       %[0-9a-f][0-9a-f]|
+                       [\x24&\x2b,:;=])*@)?
+                 (?:(?:(?:[0-9a-z]|[0-9a-z][\x2d0-9a-z]*[0-9a-z])\x2e)*
+                    (?:[a-z]|[a-z][\x2d0-9a-z]*[0-9a-z])\x2e?|
+                    \d+\x2e\d+\x2e\d+\x2e\d+)(?::\d*)?)?|
+              (?:[!'-\x2a\x2d\x2e0-9_a-z~]|%[0-9a-f][0-9a-f]|[\x24&\x2b,:;=@])+)
+           (?:/(?:[!'-\x2a\x2d\x2e0-9_a-z~]|%[0-9a-f][0-9a-f]|[\x24&\x2b,:=@])*
+              (?:;
+                 (?:[!'-\x2a\x2d\x2e0-9_a-z~]|
+                    %[0-9a-f][0-9a-f]|
+                    [\x24&\x2b,:=@])*)*
+              (?:/
+                 (?:[!'-\x2a\x2d\x2e0-9_a-z~]|%[0-9a-f][0-9a-f]|[\x24&\x2b,:=@])*
+                 (?:;
+                    (?:[!'-\x2a\x2d\x2e0-9_a-z~]|
+                       %[0-9a-f][0-9a-f]|
+                       [\x24&\x2b,:=@])*)*)*)?|
+           /(?:[!'-\x2a\x2d\x2e0-9_a-z~]|%[0-9a-f][0-9a-f]|[\x24&\x2b,:=@])*
+           (?:;(?:[!'-\x2a\x2d\x2e0-9_a-z~]|%[0-9a-f][0-9a-f]|[\x24&\x2b,:=@])*)*
+           (?:/(?:[!'-\x2a\x2d\x2e0-9_a-z~]|%[0-9a-f][0-9a-f]|[\x24&\x2b,:=@])*
+              (?:;
+                 (?:[!'-\x2a\x2d\x2e0-9_a-z~]|
+                    %[0-9a-f][0-9a-f]|
+                    [\x24&\x2b,:=@])*)*)*|
+           (?:[!'-\x2a\x2d\x2e0-9_a-z~]|%[0-9a-f][0-9a-f]|[\x24&\x2b,;=@])+
+           (?:/(?:[!'-\x2a\x2d\x2e0-9_a-z~]|%[0-9a-f][0-9a-f]|[\x24&\x2b,:=@])*
+              (?:;
+                 (?:[!'-\x2a\x2d\x2e0-9_a-z~]|
+                    %[0-9a-f][0-9a-f]|
+                    [\x24&\x2b,:=@])*)*
+              (?:/
+                 (?:[!'-\x2a\x2d\x2e0-9_a-z~]|%[0-9a-f][0-9a-f]|[\x24&\x2b,:=@])*
+                 (?:;
+                    (?:[!'-\x2a\x2d\x2e0-9_a-z~]|
+                       %[0-9a-f][0-9a-f]|
+                       [\x24&\x2b,:=@])*)*)*)?)
+        (?:\x3f(?:[!\x24&-;=\x3f@_a-z~]|%[0-9a-f][0-9a-f])*)?)?
+     (?:\x23(?:[!\x24&-;=\x3f@_a-z~]|%[0-9a-f][0-9a-f])*)?}xi
+
+
+Following example contains right-recursion:
+
+  pp ABNF.regexp_tree(<<'End')
+    s0 = n0 s0 / n1 s2 / n2 s1 / ""
+    s1 = n0 s1 / n1 s0 / n2 s2
+    s2 = n0 s2 / n1 s1 / n2 s0
+    n0 = "0" / "3" / "6" / "9"
+    n1 = "1" / "4" / "7"
+    n2 = "2" / "5" / "8"
+  End
+
+The above ABNF description represents decimal numbers
+which are multiples of 3 and the result is follows:
+
+  %r{(?:[0369]|
+        [147](?:[0369]|[147][0369]*[258])*(?:[147][0369]*[147]|[258])|
+        [258][0369]*
+        (?:[147]|
+           [258](?:[0369]|[147][0369]*[258])*(?:[147][0369]*[147]|[258])))*}xi
+
+A converted regexp can be used to match in place as:
+
+  p /\A#{ABNF.regexp <<'End'}\z/o =~ "::13.1.68.3"
+    IPv6address = "::"                  /
+          7( hex4 ":" )          hex4   /
+        1*8( hex4 ":" )      ":"        /
+          7( hex4 ":" )    ( ":" hex4 ) /
+          6( hex4 ":" ) 1*2( ":" hex4 ) /
+          5( hex4 ":" ) 1*3( ":" hex4 ) /
+          4( hex4 ":" ) 1*4( ":" hex4 ) /
+          3( hex4 ":" ) 1*5( ":" hex4 ) /
+          2( hex4 ":" ) 1*6( ":" hex4 ) /
+           ( hex4 ":" ) 1*7( ":" hex4 ) /
+                  ":"   1*8( ":" hex4 ) /
+          6( hex4 ":" )                     IPv4address /
+          6( hex4 ":" ) ":"                 IPv4address /
+          5( hex4 ":" ) ":" 0*1( hex4 ":" ) IPv4address /
+          4( hex4 ":" ) ":" 0*2( hex4 ":" ) IPv4address /
+          3( hex4 ":" ) ":" 0*3( hex4 ":" ) IPv4address /
+          2( hex4 ":" ) ":" 0*4( hex4 ":" ) IPv4address /
+           ( hex4 ":" ) ":" 0*5( hex4 ":" ) IPv4address /
+                  "::"      0*6( hex4 ":" ) IPv4address
+    IPv4address = 1*3DIGIT "." 1*3DIGIT "." 1*3DIGIT "." 1*3DIGIT
+    hex4    = 1*4HEXDIG
+  End
+
+## Contributing
+
+1. Fork it
+2. Create your feature branch (`git checkout -b my-new-feature`)
+3. Commit your changes (`git commit -am 'Add some feature'`)
+4. Push to the branch (`git push origin my-new-feature`)
+5. Create new Pull Request

data/Rakefile

@@ -0,0 +1,18 @@
+require "bundler/gem_tasks"
+
+file "lib/abnf/parser.rb" => ["lib/abnf/parser.y"] do |t|
+  sh "racc -E -o lib/abnf/parser.rb -v lib/abnf/parser.y"
+end
+
+task :build_parser => ["lib/abnf/parser.rb"]
+
+require 'rake/testtask'
+
+Rake::TestTask.new do |t|
+  t.libs << "lib"
+  t.test_files = FileList['test/*_test.rb']
+  t.ruby_opts = ['-r./test/test_helper.rb']
+  t.verbose = true
+end
+
+task :default => [:build_parser, :test]

data/abnf.gemspec

@@ -0,0 +1,24 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'abnf/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = "abnf"
+  spec.version       = Abnf::VERSION
+  spec.authors       = ["Steve Klabnik"]
+  spec.email         = ["steve@steveklabnik.com"]
+  spec.description   = %q{An Augmented Backus Naur form parser in Ruby.}
+  spec.summary       = %q{An Augmented Backus Naur form parser in Ruby.}
+  spec.homepage      = "https://github.com/steveklabnik/abnf"
+  spec.license       = "MIT"
+
+  spec.files         = `git ls-files`.split($/)
+  spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
+  spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
+  spec.require_paths = ["lib"]
+
+  spec.add_development_dependency "bundler", "~> 1.3"
+  spec.add_development_dependency "rake"
+  spec.add_development_dependency "racc"
+end

data/lib/abnf.rb

@@ -0,0 +1,57 @@
+=begin
+= ABNF
+
+convert ABNF to Regexp.
+
+== Example
+
+  # IPv6 [RFC2373]
+  p %r{\A#{ABNF.regexp <<'End'}\z}o =~ "FEDC:BA98:7654:3210:FEDC:BA98:7654:3210"
+    IPv6address = hexpart [ ":" IPv4address ]
+    IPv4address = 1*3DIGIT "." 1*3DIGIT "." 1*3DIGIT "." 1*3DIGIT
+    hexpart = hexseq | hexseq "::" [ hexseq ] | "::" [ hexseq ]
+    hexseq  = hex4 *( ":" hex4)
+    hex4    = 1*4HEXDIG
+  End
+
+Note that this is wrong because it doesn't match to "::13.1.68.3".
+
+  # URI-reference [RFC2396]
+  p %r{\A#{ABNF.regexp <<'End'}\z}o
+        URI-reference = [ absoluteURI | relativeURI ] [ "#" fragment ]
+        ...
+        digit    = "0" | "1" | "2" | "3" | "4" | "5" | "6" | "7" |
+                   "8" | "9"
+  End
+
+== ABNF
+
+=== class methods
+
+--- ABNF.regexp(abnf_description, start_symbol=nil)
+    converts ((|abnf_description|)) to a Regexp object corresponding with
+    ((|start_symbol|)).
+
+    If ((|start_symbol|)) is not specified, first symbol in
+    ((|abnf_description|)) is used.
+
+--- ABNF.regexp_tree(abnf_description, start_symbol=nil)
+    converts ((|abnf_description|)) to a ((<RegexpTree|URL:regexptree.html>)) object corresponding with
+    ((|start_symbol|)).
+
+=== constants
+
+--- TooComplex
+    is a subclass of StandardError.
+    It is raised when ABNF grammar is too complex to convert to Regexp.
+
+= Note
+
+* Wrong ABNF description produces wrong regexp.
+
+=end
+
+require 'abnf/abnf'
+require 'abnf/parser'
+require 'abnf/corerules'
+require 'abnf/regexp'

data/lib/abnf/abnf.rb

@@ -0,0 +1,136 @@
+require 'tsort'
+require 'abnf/grammar'
+
+class ABNF
+  def initialize
+    @names = []
+    @rules = {}
+    @start = nil
+  end
+
+  def start_symbol=(name)
+    @start = name
+  end
+
+  def start_symbol
+    return @start if @start
+    raise StandardError.new("no symbol defined") if @names.empty?
+    @names.first
+  end
+
+  def names
+    @names.dup
+  end
+
+  def merge(g)
+    g.each {|name, rhs|
+      self.add(name, rhs)
+    }
+  end
+
+  def [](name)
+    @rules[name]
+  end
+
+  def []=(name, rhs)
+    @names << name unless @rules.include? name
+    @rules[name] = rhs
+  end
+
+  def add(name, rhs)
+    if @rules.include? name
+      @rules[name] |= rhs
+    else
+      @names << name
+      @rules[name] = rhs
+    end
+  end
+
+  def include?(name)
+    @rules.include? name
+  end
+
+  def each(&block)
+    @names.each {|name|
+      yield name, @rules[name]
+    }
+  end
+
+  def delete_unreachable!(starts)
+    rules = {}
+    id_map = {}
+    stack = []
+    starts.each {|name|
+      next if id_map.include? name
+      each_strongly_connected_component_from(name, id_map, stack) {|syms|
+        syms.each {|sym|
+          rules[sym] = @rules[sym] if @rules.include? sym
+        }
+      }
+    }
+    @rules = rules
+    @names.reject! {|name| !@rules.include?(name)}
+    self
+  end
+
+  def delete_useless!(starts=nil)
+    if starts
+      starts = [starts] if Symbol === starts
+      delete_unreachable!(starts)
+    end
+
+    useful_names = {}
+    using_names = {}
+
+    @rules.each {|name, rhs|
+      useful_names[name] = true if rhs.useful?(useful_names)
+      rhs.each_var {|n|
+        (using_names[n] ||= {})[name] = true
+      }
+    }
+
+    queue = useful_names.keys
+    until queue.empty?
+      n = queue.pop
+      next unless using_names[n]
+      using_names[n].keys.each {|name|
+        if useful_names[name]
+          using_names[n].delete name
+        elsif @rules[name].useful?(useful_names)
+          using_names[n].delete name
+          useful_names[name] = true
+          queue << name
+        end
+      }
+    end
+
+    rules = {}
+    @rules.each {|name, rhs|
+      rhs = rhs.subst_var {|n| useful_names[n] ? nil : EmptySet}
+      rules[name] = rhs unless rhs.empty_set?
+    }
+
+    #xxx: raise if some of start symbol becomes empty set?
+
+    @rules = rules
+    @names.reject! {|name| !@rules.include?(name)}
+    self
+  end
+
+  class Alt; def useful?(useful_names) @elts.any? {|e| e.useful?(useful_names)} end end
+  class Seq; def useful?(useful_names) @elts.all? {|e| e.useful?(useful_names)} end end
+  class Rep; def useful?(useful_names) @min == 0 ? true : @elt.useful?(useful_names) end end
+  class Var; def useful?(useful_names) useful_names[@name] end end
+  class Term; def useful?(useful_names) true end end
+
+  include TSort
+  def tsort_each_node(&block)
+    @names.each(&block)
+  end
+  def tsort_each_child(name)
+    return unless @rules.include? name
+    @rules.fetch(name).each_var {|n| yield n}
+  end
+
+  class ABNFError < StandardError; end
+end