abnf-parser 0.8.1 → 0.9.0

data/lib/abnf/parser/compiler/group.rb

@@ -0,0 +1,19 @@
+module ABNF
+  module Parser
+    class Compiler
+      class Group < RuleBuilder
+        attr_accessor :rule
+
+        def group_stop _
+          compiler.pop rule
+        end
+
+        def start_rule token
+          compiler.push Alternation do |rule|
+            self.rule = rule
+          end
+        end
+      end
+    end
+  end
+end

data/lib/abnf/parser/compiler/option.rb

@@ -0,0 +1,19 @@
+module ABNF
+  module Parser
+    class Compiler
+      class Option < RuleBuilder
+        attr_accessor :rule
+
+        def option_stop _
+          compiler.pop rule
+        end
+
+        def start_rule token
+          compiler.push Alternation do |rule|
+            self.rule = rule
+          end
+        end
+      end
+    end
+  end
+end

data/lib/abnf/parser/compiler/repetition.rb

@@ -0,0 +1,50 @@
+module ABNF
+  module Parser
+    class Compiler
+      class Repetition < RuleBuilder
+        attr_writer :abnf
+        attr_accessor :element
+        attr_accessor :range
+
+        def abnf
+          @abnf ||= ''
+        end
+
+        def finished element
+          abnf << element.abnf
+
+          if range
+            rule = Rules::Repetition.new element, range, abnf
+          else
+            rule = element
+          end
+
+          compiler.pop rule
+        end
+
+        def repeat_exact token
+          number = token.lexeme.to_i
+
+          self.abnf = token.abnf
+          self.range = (number..number)
+        end
+
+        def repeat_range token
+          minimum, maximum = token.lexeme.split '*'
+
+          if minimum then minimum = minimum.to_i else minimum = 0 end
+          if maximum then maximum = maximum.to_i else maximum = Float::INFINITY end
+
+          self.abnf = token.abnf
+          self.range = (minimum..maximum)
+        end
+
+        def start_rule token
+          compiler.push Element do |element|
+            finished element
+          end
+        end
+      end
+    end
+  end
+end

data/lib/abnf/parser/compiler/rule_builder.rb

@@ -0,0 +1,32 @@
+module ABNF
+  module Parser
+    class Compiler
+      class RuleBuilder
+        attr_reader :compiler
+
+        def initialize compiler
+          @compiler = compiler
+        end
+
+        def call token
+          send token.underscore_type, token
+        end
+
+        def start_rule token
+          fail
+        end
+
+        %i(
+          prose_val hex_val_range hex_val_sequence dec_val_range
+          dec_val_sequence bin_val_range bin_val_sequence char_val group_start
+          option_start repeat_range repeat_exact rulename
+        ).each do |token_type|
+          define_method token_type do |token|
+            start_rule token
+            compiler.handle token
+          end
+        end
+      end
+    end
+  end
+end

data/lib/abnf/parser/compiler/rule_list.rb

@@ -0,0 +1,67 @@
+module ABNF
+  module Parser
+    class Compiler
+      class RuleList < RuleBuilder
+        attr_accessor :rulename
+
+        def c_nl token
+        end
+
+        def coerce_alternation rule
+          if rule.is_a? Rules::Alternation
+            rule
+          else
+            Rules::Alternation.new [rule], rule.abnf
+          end
+        end
+
+        def handle_rule name, rule
+          actual_rule = rule.inner_rule
+
+          if rule.operator == '=/'
+            original_rule = coerce_alternation rule_list[name]
+            new_rule = coerce_alternation actual_rule
+
+            alternatives = original_rule.alternatives + new_rule.alternatives
+            abnf = alternatives.map &:abnf
+            abnf *= ' / '
+            actual_rule = Rules::Alternation.new alternatives, abnf
+          end
+
+          rule_list[name] = actual_rule
+        end
+
+        def rule_list
+          compiler.rule_list
+        end
+
+        def rulename token
+          rulename = token.lexeme
+
+          compiler.push Rule do |rule|
+            handle_rule rulename, rule
+          end
+        end
+
+        class Rule < RuleBuilder
+          attr_accessor :operator
+          attr_accessor :inner_rule
+
+          def c_wsp _
+          end
+
+          def defined_as token
+            self.operator = token.lexeme
+          end
+
+          def start_rule token
+            compiler.push Alternation do |rule|
+              self.inner_rule = rule
+              compiler.pop self
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/abnf/parser/compiler/token.rb

@@ -0,0 +1,34 @@
+module ABNF
+  module Parser
+    class Compiler
+      class Token
+        attr_reader :abnf
+        attr_reader :type
+
+        def initialize type, abnf=nil
+          @abnf = abnf
+          @type = type
+        end
+
+        def == other_token
+          return false unless other_token.respond_to? :abnf
+          return false unless other_token.respond_to? :type
+
+          self.abnf == other_token.abnf and self.type == other_token.type
+        end
+
+        def lexeme
+          abnf
+        end
+
+        def inspect
+          "(#{type} #{abnf.inspect})"
+        end
+
+        def underscore_type
+          @underscore_type ||= type.downcase.gsub('-', '_')
+        end
+      end
+    end
+  end
+end

data/lib/abnf/parser/compiler/tokenizer.rb

@@ -0,0 +1,95 @@
+module ABNF
+  module Parser
+    class Compiler
+      class Tokenizer
+        C_NL = "(?:;[[:graph:][:blank:]]*)?\\r\\n".freeze
+        C_WSP = '(?:(?:#{C_NL})?[[:blank:]])'.freeze
+
+        attr_reader :abnf
+
+        def initialize abnf
+          @abnf = abnf
+        end
+
+        def self.build abnf
+          new abnf.dup
+        end
+
+        def self.call abnf
+          instance = build abnf
+          instance.()
+        end
+
+        def apply regexp
+          match_data = regexp.match abnf
+          return if match_data.nil? or not match_data.pre_match.empty?
+          abnf.slice! 0, match_data.to_s.size
+          match_data
+        end
+
+        def call &block
+          tokens = []
+
+          each do |token|
+            tokens << token
+          end
+
+          tokens
+        end
+
+        def each &block
+          until abnf.empty?
+            length = abnf.bytesize
+
+            token = self.next
+            block.(token) if block
+
+            fail "Parse error" unless abnf.bytesize < length
+          end
+        end
+
+        def next
+          possible_tokens.each do |type, regexp|
+            match_data = apply regexp
+            next unless match_data
+
+            token = Token.new type, match_data.to_s
+            return token
+          end
+
+          nil
+        end
+
+        def possible_tokens
+          @@possible_tokens ||= {
+            'prose-val'.freeze         => %r{\A<[\x20-\x3D\x3F-\x7E]*>}n,
+            'hex-val-RANGE'.freeze     => %r{\A%x[[:xdigit:]]+-[[:xdigit:]]+}n,
+            'hex-val-SEQUENCE'.freeze  => %r{\A%x[[:xdigit:]]+(?:\.[[:xdigit:]]+)*}n,
+            'dec-val-RANGE'.freeze     => %r{\A%d[[:digit:]]+-[[:digit:]]+}n,
+            'dec-val-SEQUENCE'.freeze  => %r{\A%d[[:digit:]]+(?:\.[[:digit:]]+)*}n,
+            'bin-val-RANGE'.freeze     => %r{\A%b[01]+-[01]+}n,
+            'bin-val-SEQUENCE'.freeze  => %r{\A%b[01]+(?:\.[01]+)*}n,
+            'char-val'.freeze          => %r{\A"[\x20-\x21\x23-\x7E]*"}n,
+            'option-START'.freeze      => %r{\A\[#{C_WSP}*}n,
+            'option-STOP'.freeze       => %r{\A#{C_WSP}*\]}n,
+            'group-START'.freeze       => %r{\A\(#{C_WSP}*}n,
+            'group-STOP'.freeze        => %r{\A#{C_WSP}*\)}n,
+            'repeat-RANGE'.freeze      => %r{\A[[:digit:]]*\*[[:digit:]]*}n,
+            'repeat-EXACT'.freeze      => %r{\A[[:digit:]]+}n,
+            'alternation-SLASH'.freeze => %r{\A#{C_WSP}*/#{C_WSP}*}n,
+            'defined-as'.freeze        => %r{\A#{C_WSP}*=/?#{C_WSP}*}n,
+            'rulename'.freeze          => %r{\A[[:alpha:]][-[:alnum:]]*}n,
+            'c-wsp'.freeze             => %r{\A(?:(?:#{C_NL})?[[:blank:]])+}n,
+            'c-nl'.freeze              => %r{\A#{C_NL}}n,
+          }
+        end
+
+        def to_enum
+          Enumerator::Lazy.new self do |yielder, token|
+            yielder << token
+          end
+        end
+      end
+    end
+  end
+end

data/lib/abnf/parser/controls.rb

@@ -1,4 +1,6 @@
-require 'abnf/parser/controls/ast'
-require 'abnf/parser/controls/grammar'
+require 'abnf/parser/controls/abnf'
+require 'abnf/parser/controls/nodes'
+require 'abnf/parser/controls/rule_lists'
+require 'abnf/parser/controls/rule_lists/multiples_of_three'
 require 'abnf/parser/controls/rules'
-require 'abnf/parser/controls/text_stream'
+require 'abnf/parser/controls/compiler/tokens'

data/lib/abnf/parser/controls/abnf.rb

@@ -0,0 +1,237 @@
+module ABNF
+  module Parser
+    module Controls
+      module ABNF
+        extend self
+
+        def alternation
+          %{#{ABNF.foo} / #{ABNF.bar} / #{ABNF.baz}}
+        end
+
+        def bar
+          '%x62.61.72'
+        end
+
+        def baz
+          '%x62.61.7A'
+        end
+
+        def concatenation
+          %{"[" #{foo} "]"}
+        end
+
+        def foo
+          '%x66.6F.6F'
+        end
+        alias_method :terminal, :foo
+
+        def foobar
+          '%x66.6F.6F.62.61.72'
+        end
+
+        def example
+          '(1*ALPHA "-" 1*ALPHA) / "~"'
+        end
+        alias_method :regexp_pattern, :example
+
+        def recursion
+          <<-ABNF
+s0 = n0 s0 / n1 s2 / n2 s1 / ""\r
+s1 = n0 s1 / n1 s0 / n2 s2\r
+s2 = n0 s2 / n1 s1 / n2 s0\r
+n0 = "0" / "3" / "6" / "9"\r
+n1 = "1" / "4" / "7"\r
+n2 = "2" / "5" / "8"\r
+          ABNF
+        end
+
+        def rfc5234
+          @rfc5234 ||= <<-ABNF
+rulelist       =  1*( rule / (*c-wsp c-nl) )\r
+\r
+rule           =  rulename defined-as elements c-nl\r
+                       ; continues if next line starts\r
+                       ;  with white space\r
+\r
+rulename       =  ALPHA *(ALPHA / DIGIT / "-")\r
+\r
+defined-as     =  *c-wsp ("=" / "=/") *c-wsp\r
+                       ; basic rules definition and\r
+                       ;  incremental alternatives\r
+\r
+elements       =  alternation *c-wsp\r
+\r
+c-wsp          =  WSP / (c-nl WSP)\r
+\r
+c-nl           =  comment / CRLF\r
+                       ; comment or newline\r
+\r
+comment        =  ";" *(WSP / VCHAR) CRLF\r
+\r
+alternation    =  concatenation\r
+                  *(*c-wsp "/" *c-wsp concatenation)\r
+\r
+concatenation  =  repetition *(1*c-wsp repetition)\r
+\r
+repetition     =  [repeat] element\r
+\r
+repeat         =  1*DIGIT / (*DIGIT "*" *DIGIT)\r
+\r
+element        =  rulename / group / option /\r
+                  char-val / num-val / prose-val\r
+\r
+group          =  "(" *c-wsp alternation *c-wsp ")"\r
+\r
+option         =  "[" *c-wsp alternation *c-wsp "]"\r
+\r
+char-val       =  DQUOTE *(%x20-21 / %x23-7E) DQUOTE\r
+                       ; quoted string of SP and VCHAR\r
+                       ;  without DQUOTE\r
+\r
+num-val        =  "%" (bin-val / dec-val / hex-val)\r
+\r
+bin-val        =  "b" 1*BIT\r
+                  [ 1*("." 1*BIT) / ("-" 1*BIT) ]\r
+                       ; series of concatenated bit values\r
+                       ;  or single ONEOF range\r
+\r
+dec-val        =  "d" 1*DIGIT\r
+                  [ 1*("." 1*DIGIT) / ("-" 1*DIGIT) ]\r
+\r
+hex-val        =  "x" 1*HEXDIG\r
+                  [ 1*("." 1*HEXDIG) / ("-" 1*HEXDIG) ]\r
+\r
+prose-val      =  "<" *(%x20-3D / %x3F-7E) ">"\r
+                       ; bracketed string of SP and VCHAR\r
+                       ;  without angles\r
+                       ; prose description, to be used as\r
+                       ;  last resort\r
+          ABNF
+        end
+
+        def rfc7230
+          @rfc7230 ||= <<-ABNF
+BWS = OWS\r
+\r
+Connection = *( "," OWS ) connection-option *( OWS "," [ OWS\r
+ connection-option ] )\r
+\r
+Content-Length = 1*DIGIT\r
+\r
+HTTP-message = start-line *( header-field CRLF ) CRLF [ message-body\r
+ ]\r
+HTTP-name = %x48.54.54.50 ; HTTP\r
+HTTP-version = HTTP-name "/" DIGIT "." DIGIT\r
+Host = uri-host [ ":" port ]\r
+\r
+OWS = *( SP / HTAB )\r
+\r
+RWS = 1*( SP / HTAB )\r
+\r
+TE = [ ( "," / t-codings ) *( OWS "," [ OWS t-codings ] ) ]\r
+Trailer = *( "," OWS ) field-name *( OWS "," [ OWS field-name ] )\r
+Transfer-Encoding = *( "," OWS ) transfer-coding *( OWS "," [ OWS\r
+ transfer-coding ] )\r
+\r
+URI-reference = <URI-reference, see [RFC3986], Section 4.1>\r
+Upgrade = *( "," OWS ) protocol *( OWS "," [ OWS protocol ] )\r
+\r
+Via = *( "," OWS ) ( received-protocol RWS received-by [ RWS comment\r
+ ] ) *( OWS "," [ OWS ( received-protocol RWS received-by [ RWS\r
+ comment ] ) ] )\r
+\r
+absolute-URI = <absolute-URI, see [RFC3986], Section 4.3>\r
+absolute-form = absolute-URI\r
+absolute-path = 1*( "/" segment )\r
+asterisk-form = "*"\r
+authority = <authority, see [RFC3986], Section 3.2>\r
+authority-form = authority\r
+\r
+chunk = chunk-size [ chunk-ext ] CRLF chunk-data CRLF\r
+chunk-data = 1*OCTET\r
+chunk-ext = *( ";" chunk-ext-name [ "=" chunk-ext-val ] )\r
+chunk-ext-name = token\r
+chunk-ext-val = token / quoted-string\r
+chunk-size = 1*HEXDIG\r
+chunked-body = *chunk last-chunk trailer-part CRLF\r
+comment = "(" *( ctext / quoted-pair / comment ) ")"\r
+connection-option = token\r
+ctext = HTAB / SP / %x21-27 ; '!'-'''\r
+ / %x2A-5B ; '*'-'['\r
+ / %x5D-7E ; ']'-'~'\r
+ / obs-text\r
+\r
+field-content = field-vchar [ 1*( SP / HTAB ) field-vchar ]\r
+field-name = token\r
+field-value = *( field-content / obs-fold )\r
+field-vchar = VCHAR / obs-text\r
+fragment = <fragment, see [RFC3986], Section 3.5>\r
+\r
+header-field = field-name ":" OWS field-value OWS\r
+http-URI = "http://" authority path-abempty [ "?" query ] [ "#"\r
+ fragment ]\r
+https-URI = "https://" authority path-abempty [ "?" query ] [ "#"\r
+ fragment ]\r
+\r
+last-chunk = 1*"0" [ chunk-ext ] CRLF\r
+\r
+message-body = *OCTET\r
+method = token\r
+\r
+obs-fold = CRLF 1*( SP / HTAB )\r
+obs-text = %x80-FF\r
+origin-form = absolute-path [ "?" query ]\r
+\r
+partial-URI = relative-part [ "?" query ]\r
+path-abempty = <path-abempty, see [RFC3986], Section 3.3>\r
+port = <port, see [RFC3986], Section 3.2.3>\r
+protocol = protocol-name [ "/" protocol-version ]\r
+protocol-name = token\r
+protocol-version = token\r
+pseudonym = token\r
+\r
+qdtext = HTAB / SP / "!" / %x23-5B ; '#'-'['\r
+ / %x5D-7E ; ']'-'~'\r
+ / obs-text\r
+query = <query, see [RFC3986], Section 3.4>\r
+quoted-pair = "\" ( HTAB / SP / VCHAR / obs-text )\r
+\r
+quoted-string = DQUOTE *( qdtext / quoted-pair ) DQUOTE\r
+\r
+rank = ( "0" [ "." *3DIGIT ] ) / ( "1" [ "." *3"0" ] )\r
+reason-phrase = *( HTAB / SP / VCHAR / obs-text )\r
+received-by = ( uri-host [ ":" port ] ) / pseudonym\r
+received-protocol = [ protocol-name "/" ] protocol-version\r
+relative-part = <relative-part, see [RFC3986], Section 4.2>\r
+request-line = method SP request-target SP HTTP-version CRLF\r
+request-target = origin-form / absolute-form / authority-form /\r
+ asterisk-form\r
+\r
+scheme = <scheme, see [RFC3986], Section 3.1>\r
+segment = <segment, see [RFC3986], Section 3.3>\r
+start-line = request-line / status-line\r
+status-code = 3DIGIT\r
+status-line = HTTP-version SP status-code SP reason-phrase CRLF\r
+\r
+t-codings = "trailers" / ( transfer-coding [ t-ranking ] )\r
+t-ranking = OWS ";" OWS "q=" rank\r
+tchar = "!" / "#" / "$" / "%" / "&" / "'" / "*" / "+" / "-" / "." /\r
+ "^" / "_" / "`" / "|" / "~" / DIGIT / ALPHA\r
+token = 1*tchar\r
+trailer-part = *( header-field CRLF )\r
+transfer-coding = "chunked" / "compress" / "deflate" / "gzip" /\r
+ transfer-extension\r
+transfer-extension = token *( OWS ";" OWS transfer-parameter )\r
+transfer-parameter = token BWS "=" BWS ( token / quoted-string )\r
+\r
+uri-host = <host, see [RFC3986], Section 3.2.2>\r
+          ABNF
+        end
+
+        def rule
+          "some-rule = #{foo}\r\n"
+        end
+      end
+    end
+  end
+end