ebnf 0.3.9 → 1.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: b0a5b5d879cad3c68665f4cc360299ec06934f91
-  data.tar.gz: 3310fb9e5dc956f6d51cde5d6a4cc816ef78fc4e
+  metadata.gz: 80ecfa0591fe3d22e557724d70f4ba384f0809bd
+  data.tar.gz: 96c0de9edaa30e4d8a063cc9f635a2157d85ba7e
 SHA512:
-  metadata.gz: c4f1450e2035d5f0457b98a00d7e89b4a397f6899d16fa0bcfa78c4cb88d287eaff5e4ef5435c2a06612b3d972b544770300ce1608f743215cde3aee0b275385
-  data.tar.gz: a44d8ed56897dcb5eb224c7dba62d5876e50e979eb0d95459fa7cd2714626d4414f11d27b1661f3fefe755766362ba7048ee3bb4256b4bc888d3fe6703e7b44c
+  metadata.gz: 5e952a2ccac175a29d4860f802eb3e5eb28ad41db4a36fca9eac98370d93b4543c12c775d698d9492ab28c876851117b9ea51f35824a040df20d06c52fb3f8b5
+  data.tar.gz: c8d2135602e8cfe704e7cb677dcea4e9ec500067a54bb41cb44f0c7fe4f99df9eba59e4a1b51717e5b7559bff6cf159e309e5224678cef1e63b3654b14c9d529

data/README.md

@@ -1,8 +1,12 @@
-# EBNF [![Build Status](https://secure.travis-ci.org/gkellogg/ebnf.png?branch=master)](http://travis-ci.org/gkellogg/ebnf)
-
+# EBNF 
 
 [EBNF][] parser and generic parser generator.
 
+[![Gem Version](https://badge.fury.io/rb/ebnf.png)](http://badge.fury.io/rb/ebnf)
+[![Build Status](https://secure.travis-ci.org/gkellogg/ebnf.png?branch=master)](http://travis-ci.org/gkellogg/ebnf)
+[![Coverage Status](https://coveralls.io/repos/gkellogg/ebnf/badge.svg)](https://coveralls.io/r/gkellogg/ebnf)
+[![Dependency Status](https://gemnasium.com/gkellogg/ebnf.png)](https://gemnasium.com/gkellogg/ebnf)
+
 ## Description
 This is a [Ruby][] implementation of an [EBNF][] and [BNF][] parser and parser generator.
 It parses [EBNF][] grammars to [BNF][], generates [First/Follow and Branch][] tables for

data/VERSION

@@ -1 +1 @@
-0.3.9
+1.0.0

data/etc/ebnf.ebnf

@@ -45,7 +45,7 @@
     # Strings are unescaped Unicode, excepting control characters and hash (#)
     [16] STRING1    ::= '"' (CHAR - '"')* '"'
 
-    [17] STRING2    ::= "'" (CHAR - "'"))* "'"
+    [17] STRING2    ::= "'" (CHAR - "'")* "'"
 
     [18] CHAR       ::= HEX
                       | [#x20#x21#x22]

data/etc/ebnf.html

@@ -155,7 +155,8 @@
       <td>::=</td>
       <td>
         "<code class="grammar-literal">'</code>"
-        <code>(</code> <a href="#grammar-production-CHAR">CHAR</a> <code>-</code> "<code class="grammar-literal">'</code>"<code>)</code>
+        (<a href="#grammar-production-CHAR">CHAR</a> <code>-</code> "<code class="grammar-literal">'</code>")<code>*</code> 
+        "<code class="grammar-literal">'</code>"
       </td>
     </tr>
     <tr id='grammar-production-CHAR'>

data/etc/ebnf.ll1.sxp

@@ -149,7 +149,7 @@
  (terminal O_RANGE "15"
   (seq "[^" (plus (alt (seq R_BEGIN (alt HEX R_CHAR)) (alt HEX R_CHAR))) "]"))
  (terminal STRING1 "16" (seq "\"" (star (diff CHAR "\"")) "\""))
- (terminal STRING2 "17" (seq "'" (diff CHAR "'")))
+ (terminal STRING2 "17" (seq "'" (star (diff CHAR "'")) "'"))
  (terminal CHAR "18" (alt HEX (range "#x20#x21#x22") (range "#x24-#x00FFFFFF")))
  (terminal R_CHAR "19" (diff CHAR "]"))
  (terminal R_BEGIN "20" (seq (alt HEX R_CHAR) "-"))

data/etc/ebnf.rb

@@ -1,4 +1,4 @@
-# This file is automatically generated by bin/ebnf
+# This file is automatically generated by /Users/gregg/Projects/ebnf/lib/ebnf/base.rb
 # BRANCH derived from etc/ebnf.ebnf
 module Branch
   START = :ebnf

data/etc/ebnf.sxp

@@ -19,7 +19,7 @@
  (terminal O_RANGE "15"
   (seq "[^" (plus (alt (seq R_BEGIN (alt HEX R_CHAR)) (alt HEX R_CHAR))) "]"))
  (terminal STRING1 "16" (seq "\"" (star (diff CHAR "\"")) "\""))
- (terminal STRING2 "17" (seq "'" (diff CHAR "'")))
+ (terminal STRING2 "17" (seq "'" (star (diff CHAR "'")) "'"))
  (terminal CHAR "18" (alt HEX (range "#x20#x21#x22") (range "#x24-#x00FFFFFF")))
  (terminal R_CHAR "19" (diff CHAR "]"))
  (terminal R_BEGIN "20" (seq (alt HEX R_CHAR) "-"))

data/lib/ebnf/ll1/lexer.rb

@@ -113,6 +113,8 @@ module EBNF::LL1
     # @param  [Hash{Symbol => Object}]        options
     # @option options [Regexp]                :whitespace
     #   Whitespace between tokens, including comments
+    # @option options[Integer] :high_water passed to scanner
+    # @option options[Integer] :low_water passed to scanner
     def initialize(input = nil, terminals = nil, options = {})
       @options        = options.dup
       @whitespace     = @options[:whitespace]
@@ -123,7 +125,7 @@ module EBNF::LL1
       raise Error, "Terminal patterns not defined" unless @terminals && @terminals.length > 0
 
       @lineno = 1
-      @scanner = Scanner.new(input)
+      @scanner = Scanner.new(input, options)
     end
 
     ##
@@ -261,6 +263,10 @@ module EBNF::LL1
     def match_token
       @terminals.each do |term|
         #STDERR.puts "match[#{term.type}] #{scanner.rest[0..100].inspect} against #{term.regexp.inspect}" #if term.type == :STRING_LITERAL_SINGLE_QUOTE
+        if term.partial_regexp && scanner.match?(term.partial_regexp) && !scanner.match?(term.regexp)
+          scanner.ensure_buffer_full
+        end
+
         if matched = scanner.scan(term.regexp)
           #STDERR.puts "  matched #{term.type.inspect}: #{matched.inspect}"
           tok = token(term.type, term.canonicalize(matched))
@@ -278,6 +284,7 @@ module EBNF::LL1
     class Terminal
       attr_reader :type
       attr_reader :regexp
+      attr_reader :partial_regexp
 
       # @param [Symbol, nil] type
       # @param [Regexp] regexp
@@ -287,8 +294,11 @@ module EBNF::LL1
       #   their canonical value
       # @option options [Boolean] :unescape
       #   Cause strings and codepoints to be unescaped.
+      # @option options [Regexp] :partial_regexp
+      #   A regular expression matching the beginning of this terminal; useful for terminals that match things longer than the scanner low water mark.
       def initialize(type, regexp, options = {})
         @type, @regexp, @options = type, regexp, options
+        @partial_regexp = options[:partial_regexp]
         @map = options.fetch(:map, {})
       end
       
@@ -327,8 +337,6 @@ module EBNF::LL1
 
     end
 
-  protected
-
     ##
     # Constructs a new token object annotated with the current line number.
     #

data/lib/ebnf/ll1/parser.rb

@@ -206,6 +206,8 @@ module EBNF::LL1
     #   Detailed debug output
     # @option options [Boolean] :reset_on_start
     #   Reset the parser state if the start token set with `prod` is found in a production. This reduces the production stack depth growth, which is appropriate for some grammars.
+    # @option options[Integer] :high_water passed to lexer
+    # @option options[Integer] :low_water passed to lexer
     # @yield [context, *data]
     #   Yields for to return data to parser
     # @yieldparam [:statement, :trace] context

data/lib/ebnf/ll1/scanner.rb

@@ -1,3 +1,4 @@
+# coding: utf-8
 require 'strscan'    unless defined?(StringScanner)
 
 module EBNF::LL1
@@ -9,8 +10,8 @@ module EBNF::LL1
   #
   # FIXME: Only implements the subset required by the Lexer for now.
   class Scanner < StringScanner
-    HIGH_WATER = 10240
-    LOW_WATER  = 2048     # Hopefully large enough to deal with long multi-line comments
+    HIGH_WATER = 512 * 1024     # Hopefully large enough to deal with long multi-line comments
+    LOW_WATER  = 4 * 1024
 
     ##
     # @return [IO, StringIO]
@@ -25,14 +26,14 @@ module EBNF::LL1
     # @option options[Integer] :low_water (LOW_WATER)
     # @return [Scanner]
     def initialize(input, options = {})
-      @options = options.merge(:high_water => HIGH_WATER, :low_water => LOW_WATER)
+      @options = options.merge(high_water: HIGH_WATER, low_water: LOW_WATER)
 
       if input.respond_to?(:read)
         @input = input
         super("")
         feed_me
       else
-        super(input.to_s)
+        super(encode_utf8 input.to_s)
       end
     end
 
@@ -95,12 +96,12 @@ module EBNF::LL1
       feed_me
       encode_utf8 super
     end
-    
-  private
-    # Maintain low-water mark
-    def feed_me
-      if rest_size < @options[:low_water] && @input && !@input.eof?
-        # Read up to high-water mark ensuring we're at an end of line
+
+    ##
+    # Ensures that the input buffer is full to the high water mark, or end of file. Useful when matching tokens that may be longer than the low water mark
+    def ensure_buffer_full
+      # Read up to high-water mark ensuring we're at an end of line
+      if @input && !@input.eof?
         diff = @options[:high_water] - rest_size
         string = encode_utf8(@input.read(diff))
         string << encode_utf8(@input.gets) unless @input.eof?
@@ -108,9 +109,19 @@ module EBNF::LL1
       end
     end
 
+  private
+    # Maintain low-water mark
+    def feed_me
+      ensure_buffer_full if rest_size < @options[:low_water]
+    end
+
     # Perform UTF-8 encoding of input
     def encode_utf8(string)
-      string.respond_to?(:force_encoding) ? string.force_encoding(Encoding::UTF_8) : string
+      if string && string.encoding != Encoding::UTF_8
+        string = string.dup if string.frozen?
+        string.force_encoding(Encoding::UTF_8)
+      end
+      string
     end
   end
 end

data/lib/ebnf/rule.rb

@@ -142,7 +142,7 @@ module EBNF
     # Serializes this rule to an Turtle
     # @return [String]
     def to_ttl
-      @ebnf.debug("to_ttl") {inspect}
+      @ebnf.debug("to_ttl") {inspect} if @ebnf
       comment = orig.strip.
         gsub(/"""/, '\"\"\"').
         gsub("\\", "\\\\").

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: ebnf
 version: !ruby/object:Gem::Version
-  version: 0.3.9
+  version: 1.0.0
 platform: ruby
 authors:
 - Gregg Kellogg
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-05-01 00:00:00.000000000 Z
+date: 2015-10-30 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: sxp
@@ -175,7 +175,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.4.3
+rubygems_version: 2.4.5.1
 signing_key: 
 specification_version: 4
 summary: EBNF parser and parser generator.