fabulator-grammar 0.0.1 → 0.0.3

data/History.txt

@@ -0,0 +1,22 @@
+=== 0.0.3 2010-09-07
+
+* 5 major enhancements:
+  * structural elements in place for writing grammars
+  * limited rule matching support
+  * use [,] for repetition counts
+  * character set algrebra supported
+  * pre-defined character sets
+  * requires Fabulator 0.0.7 or higher
+
+=== 0.0.2 2010-08-19
+
+* 3 minor enhancements:
+  * Create file that keeps config.gem from complaining
+  * Change character sets to look a little like Perl 6
+  * Use <,> for repetition counts
+
+=== 0.0.1 2010-08-11
+
+* 1 major enhancement:
+  * Initial release
+

data/Rakefile

@@ -7,8 +7,10 @@ begin
     gem.email = "jgsmith@tamu.edu"
     gem.homepage = "http://github.com/jgsmith/ruby-fabulator-grammar"
     gem.authors = ["James Smith"]
-    gem.add_dependency(%q<fabulator>, [">= 0.0.1"]) 
+    gem.add_dependency(%q<fabulator>, [">= 0.0.7"]) 
     # gem is a Gem::Specification... see http://www.rubygems.org/read/chapter/20 for additional settings
+    # not sure how to add dependency of a library that's not a gem
+    gem.requirements << 'bitset, 1.0 or greater'
   end
 rescue LoadError
   puts "Jeweler (or a dependency) not available. This is only required if you plan to package fabulator-exhibit as a gem."

data/VERSION

@@ -1 +1 @@
-0.0.1
+0.0.3

data/features/grammar.feature

@@ -23,7 +23,7 @@ Feature: Basic regex parsing
   Scenario: Parsing a simple text string
    Given a context
      And the prefix g as "http://dh.tamu.edu/ns/fabulator/grammar/1.0#"
-   When I parse the regex (foo{1,4}$)
+   When I parse the regex (foo[1,4]$)
    Then it should match "foo"
    Then it should match "fooooo"
    Then it should not match "foooooo"
@@ -33,7 +33,7 @@ Feature: Basic regex parsing
   Scenario: Parsing a simple text string
    Given a context
      And the prefix g as "http://dh.tamu.edu/ns/fabulator/grammar/1.0#"
-   When I parse the regex (^[Ff]o[-a-zA-F01234-9]+?)
+   When I parse the regex (^[ [Ff] ]o[ [-a-zA-F01234-9] ]+?)
    Then it should match "foo"
     And it should match "Foo"
     And it should match "FoF03z-"
@@ -43,33 +43,45 @@ Feature: Basic regex parsing
   Scenario: Parsing a simple text string
    Given a context
      And the prefix g as "http://dh.tamu.edu/ns/fabulator/grammar/1.0#"
-   When I parse the regex (^[^0-9][a-z][0-9]$)
-   Then it should match "fo0"
-    And it should not match "0l0"
+   When I parse the regex (^[ [Ff] ]o[ [-a-z] + [A-F] + [0-9] - [5] ]+?)
+   Then it should match "foo"
+    And it should match "Foo"
+    And it should match "FoF03z-"
+    And it should not match "hellofoo"
+    And it should not match "hellof5o"
 
+  @chars
   Scenario: Parsing a simple text string
    Given a context
      And the prefix g as "http://dh.tamu.edu/ns/fabulator/grammar/1.0#"
-   When I parse the regex (^[^0-9].[0-9]$)
-   Then it should match "fo0"
-    And it should not match "0l0"
-    And it should not match "00"
+   When I parse the regex (^[ [Ff] ]o[ [-] + :lower: + :xdigit: - [5] ]+?)
+   Then it should match "foo"
+    And it should match "Foo"
+    And it should match "FoF03z-"
+    And it should not match "hellofoo"
+    And it should not match "hellof5o"
 
   @chars
   Scenario: Parsing a simple text string
    Given a context
      And the prefix g as "http://dh.tamu.edu/ns/fabulator/grammar/1.0#"
-   When I parse the regex ([\[-\]]o[a-z\]A-F01234-9]+?)
+   When I parse the regex (^[ -[0-9] ][ :lower: ][ [0-9] ]$)
+   Then it should match "fo0"
+    And it should not match "0l0"
 
   Scenario: Parsing a simple text string
    Given a context
      And the prefix g as "http://dh.tamu.edu/ns/fabulator/grammar/1.0#"
-   When I parse the regex (<foo><g:bar>*)
+   When I parse the regex (^[-:digit:].[:digit:]$)
+   Then it should match "fo0"
+    And it should not match "0l0"
+    And it should not match "00"
 
+  @chars
   Scenario: Parsing a simple text string
    Given a context
      And the prefix g as "http://dh.tamu.edu/ns/fabulator/grammar/1.0#"
-   When I parse the regex (bar ( <foo> <g:bar> ) *)
+   When I parse the regex ([[\[-\]]]o[[a-z\]A-F01234-9]]+?)
 
   Scenario: Adding two numbers together as a union
    Given a context
@@ -86,3 +98,95 @@ Feature: Basic regex parsing
    When I run the expression (g:match('^foo', 'bfooo'))
    Then I should get 1 item
      And item 0 should be false
+
+  @rules
+  Scenario: Parsing a simple text string
+   Given a context
+   Then I can parse the rule (foo bar)
+    And I can parse the rule (foo(s) bar(?))
+    And I can parse the rule (foo(s?) bar(3))
+    And I can parse the rule (foo(s NL) bar(3..4))
+    And I can parse the rule (foo ';'(s) bar(3..4 ','))
+    And I can parse the rule ((foo bar)(s))
+    And I can not parse the rule (foo(s NL(s)))
+    And I can parse the rule (foo(s (NL NL(s))))
+    And I can not parse the rule (foo(s NL NL(s)))
+
+  @grammar
+  Scenario: Parsing a grammar xml definition
+    Given a context
+    Given the grammar
+      """
+        <g:grammar xmlns:g="http://dh.tamu.edu/ns/fabulator/grammar/1.0#">
+          <g:token g:name="LETTER" g:matches="[:alpha:]" />
+          <g:token g:name="NUMBER" g:matches="[:digit:]" />
+          <g:rule g:name="something">
+            <g:when g:matches="LETTER NUMBER LETTER" />
+          </g:rule>
+          <g:rule g:name="other">
+            <g:when g:matches="a := LETTER b := NUMBER c := LETTER" />
+          </g:rule>
+          <g:rule g:name="or">
+            <g:when g:matches="other(s) d := LETTER(s)" />
+          </g:rule>
+          <g:rule g:name="ooor">
+            <g:when g:matches="other(s ',') d := LETTER(s)" />
+          </g:rule>
+        </g:grammar>
+      """
+    Then "something" should match "a0a"
+     And "something" should not match "abc"
+     And "something" should match "ab0c"
+     And "other" should match "a0a" 
+     And "other" should match "ab1c" 
+     And "or" should match "a1bcde"
+     And "or" should match "a1bb2ccde"
+     And "or" should match "a1bwb2ccde"
+     And "or" should not parse "a1bwb2ccde"
+     And "or" should match "acbwb2ccde"
+     And "or" should not parse "acbwb2ccde"
+     And "or" should not match "acbwb2d"
+     And "ooor" should parse "a1c,b2df"
+     And the expression (d) should equal ['f']
+     And the expression (other[1]/b) should equal ['1']
+     And the expression (other[2]/b) should equal ['2']
+     And "ooor" should not parse "a1c,b2d"
+
+  @mode
+  Scenario: Parsing a grammar xml definition
+    Given a context
+    Given the grammar
+      """
+        <g:grammar xmlns:g="http://dh.tamu.edu/ns/fabulator/grammar/1.0#">
+          <g:token g:name="LETTER" g:matches="[:alpha:]" g:mode="normal"/>
+          <g:token g:name="NUMBER" g:matches="[:digit:]" />
+          <g:token g:name="LETTER" g:matches="[:upper:]" g:mode="upper"/>
+          <g:token g:name="LETTER" g:matches="[:lower:]" g:mode="lower"/>
+          <g:rule g:name="something">
+            <g:when g:matches="[mode normal] LETTER NUMBER [mode upper] LETTER" />
+          </g:rule>
+        </g:grammar>
+      """
+    Then "something" should parse "a0A"
+     And "something" should not parse "a0a"
+
+  @context
+  Scenario: Parsing a grammar xml definition
+    Given a context
+    Given the grammar
+      """
+        <g:grammar xmlns:g="http://dh.tamu.edu/ns/fabulator/grammar/1.0#">
+          <g:context g:mode="normal">
+            <g:token g:name="LETTER" g:matches="[:alpha:]" />
+          </g:context>
+          <g:token g:name="NUMBER" g:matches="[:digit:]" />
+          <g:token g:name="LETTER" g:matches="[:upper:]" g:mode="upper"/>
+          <g:token g:name="LETTER" g:matches="[:lower:]" g:mode="lower"/>
+          <g:rule g:name="something">
+            <g:when g:matches="^^ [mode normal] LETTER NUMBER [mode upper] LETTER" />
+          </g:rule>
+        </g:grammar>
+      """
+    Then "something" should match "a0A"
+     And "something" should not match "a0a"
+     And "something" should not match "aa0A"

data/features/step_definitions/expression_steps.rb

@@ -19,7 +19,6 @@ end
 Given 'a context' do
   @context ||= Fabulator::Expr::Context.new
   @parser ||= Fabulator::Expr::Parser.new
-  @grammar_parser ||= Fabulator::Grammar::Parser.new
 end
 
 Given /the prefix (\S+) as "([^"]+)"/ do |p,h|
@@ -72,7 +71,8 @@ Then /I should get the type (.*)/ do |t|
 end
 
 Then /I should get (\d+) items?/ do |count|
-  @result.length.should == count
+  #puts "result types: #{@result.collect{|r| r.class.name}.join(', ')}"
+  @result.size.should == count
 end
 
 Then /item (\d+) should be (\[.*\])/ do |i,t|

data/features/step_definitions/grammar_steps.rb

@@ -2,13 +2,30 @@ require 'yaml'
 
 When /^I parse the regex \((.*)\)$/ do |regex|
   @context ||= Fabulator::Expr::Context.new
-  @grammar_parser ||= Fabulator::Grammar::Parser.new
-  @regex = @grammar_parser.parse(regex, @context)
+  @grammar_parser ||= Fabulator::Grammar::TokenParser.new
+  @rule_parser ||= Fabulator::Grammar::RuleParser.new
+  @regex = @grammar_parser.parse(regex)
 #  puts YAML::dump(r)
 #  puts @regex.to_regex
 #  pending # express the regexp above with the code you wish you had
 end
 
+Then /^I can parse the rule \((.*)\)$/ do |rule|
+  @context ||= Fabulator::Expr::Context.new
+  @grammar_parser ||= Fabulator::Grammar::TokenParser.new
+  @rule_parser ||= Fabulator::Grammar::RuleParser.new
+  @rule = @rule_parser.parse(rule)
+  @rule.should_not == nil
+end
+
+Then /^I can not parse the rule \((.*)\)$/ do |rule|
+  @context ||= Fabulator::Expr::Context.new
+  @grammar_parser ||= Fabulator::Grammar::TokenParser.new
+  @rule_parser ||= Fabulator::Grammar::RuleParser.new
+  @rule = (@rule_parser.parse(rule) rescue nil)
+  @rule.should == nil
+end
+
 Then /^it should match "(.*)"$/ do |str|
   str.should =~ @regex.to_regex
 end
@@ -16,3 +33,30 @@ end
 Then /^it should not match "(.*)"$/ do |str|
   str.should_not =~ @regex.to_regex
 end
+
+Then /^"(.*)" should match "(.*)"$/ do |nom, str|
+  ret = @grammar.match(@context, nom, str)
+#  puts YAML::dump(ret)
+  ret.should == true
+end
+
+Then /^"(.*)" should parse "(.*)"$/ do |nom, str|
+  ret = @grammar.parse(@context, nom, str)
+  #puts YAML::dump(ret)
+  ret.should_not == nil
+
+  @context.root = @context.root.anon_node(nil)
+  if !ret.nil?
+    @context.merge_data(ret)
+    @context.root.roots['data'] = @context.root
+#puts YAML::dump(@context.root.to_h)
+  end
+end
+
+Then /^"(.*)" should not parse "(.*)"$/ do |nom, str|
+  @grammar.parse(@context, nom, str).should == nil
+end
+
+Then /^"(.*)" should not match "(.*)"$/ do |nom, str|
+  @grammar.match(@context, nom, str).should == false
+end

data/features/step_definitions/xml_steps.rb

@@ -1,23 +1,12 @@
-Given /the statemachine/ do |doc_xml|
+Given /the grammar/ do |doc_xml|
   @context ||= Fabulator::Expr::Context.new
 
-  if @sm.nil?
-    @sm = Fabulator::Core::StateMachine.new.compile_xml(doc_xml)
+  if @grammar.nil?
+    @grammar = Fabulator::Grammar::Actions::Grammar.new.compile_xml(doc_xml, @context)
   else
-    @sm.compile_xml(doc_xml)
+    @grammar.compile_xml(doc_xml, @context)
   end
-  @sm.init_context(@context)
-end
 
-When /I run it with the following params:/ do |param_table|
-  params = { }
-  param_table.hashes.each do |hash|
-    params[hash['key']] = hash['value']
-  end
-  @sm.run(params)
-  #puts YAML::dump(@sm)
+#  puts YAML::dump(@grammar)
 end
 
-Then /it should be in the '(.*)' state/ do |s|
-  @sm.state.should == s
-end

data/features/support/env.rb

@@ -1,6 +1,7 @@
 # This file makes it possible to install RubyCAS-Client as a Rails plugin.
 
 $: << File.expand_path(File.dirname(__FILE__))+'/../../lib'
+$: << File.expand_path(File.dirname(__FILE__))+'/../../../fabulator/lib'
 
 require 'fabulator'
 require 'fabulator/grammar'

data/lib/fabulator-grammar.rb

@@ -0,0 +1 @@
+require 'fabulator/grammar'

data/lib/fabulator/grammar.rb

@@ -1,12 +1,21 @@
-require 'fabulator/grammar/parser'
+require 'fabulator/grammar/rule_parser'
+require 'fabulator/grammar/token_parser'
 require 'fabulator/grammar/actions'
-require 'fabulator/grammar/expr/rules'
-require 'fabulator/grammar/expr/rule'
+require 'fabulator/grammar/expr/token'
+require 'fabulator/grammar/expr/token_alternative'
 require 'fabulator/grammar/expr/rule_ref'
 require 'fabulator/grammar/expr/text'
 require 'fabulator/grammar/expr/sequence'
+require 'fabulator/grammar/expr/set_skip'
 require 'fabulator/grammar/expr/char_set'
 require 'fabulator/grammar/expr/any'
+require 'fabulator/grammar/expr/look_ahead'
+require 'fabulator/grammar/cursor'
+require 'fabulator/grammar/expr/rule'
+require 'fabulator/grammar/expr/rule_mode'
+require 'fabulator/grammar/expr/rule_alternative'
+require 'fabulator/grammar/expr/rule_sequence'
+require 'fabulator/grammar/expr/anchor'
 
 module Fabulator
   module Grammar

data/lib/fabulator/grammar/actions.rb

@@ -1,22 +1,32 @@
-#require 'fabulator/grammar/actions/grammar'
 
 module Fabulator
   GRAMMAR_NS = "http://dh.tamu.edu/ns/fabulator/grammar/1.0#"
+
+  require 'fabulator/grammar/actions/grammar'
+  require 'fabulator/grammar/actions/context'
+  require 'fabulator/grammar/actions/rule'
+  require 'fabulator/grammar/actions/token'
+  require 'fabulator/grammar/actions/when'
+
   module Grammar
     module Actions
-      class Lib
-        include Fabulator::ActionLib
-
+      class Lib < Fabulator::TagLib
         register_namespace GRAMMAR_NS
 
-        #action 'grammar', Grammar
+        structural 'grammar', Grammar
+        structural 'context', Context
+        structural 'rule', Rule
+        structural 'token', Token
+        structural 'when', When
+
+#        action 'result', Result
 
         ## reference a grammar name
         function 'match' do |ctx, args|
           # first arg is the regex or <rule name>
           regex = args[0].to_s
-          parser = Fabulator::Grammar::Parser.new
-          compiled = parser.parse(regex, ctx).to_regex
+          parser = Fabulator::Grammar::TokenParser.new
+          compiled = parser.parse(regex).to_regex
           if args[1].is_a?(Array)
             args[1].collect{|a|
               if a.to_s =~ compiled

data/lib/fabulator/grammar/actions/context.rb

@@ -0,0 +1,18 @@
+module Fabulator
+  module Grammar
+    module Actions
+      class Context < Fabulator::Structural
+
+    namespace GRAMMAR_NS
+
+    attribute :mode, :default => :default, :static => true
+
+    attr_accessor :mode, :tokens, :rules
+
+    contains :rule
+    contains :token
+
+      end
+    end
+  end
+end

data/lib/fabulator/grammar/actions/grammar.rb

@@ -0,0 +1,76 @@
+module Fabulator
+  module Grammar
+    module Actions
+      class Grammar < Fabulator::Structural
+
+    namespace GRAMMAR_NS
+
+    contains :rule
+    contains :token
+    contains :context
+
+    has_actions
+
+    def compile_xml(xml, context = nil)
+       super
+
+      @modes = { :default => { } }
+
+      @contexts.each do |c|
+        c.tokens.each do |token|
+          self.add_rule(token, c.mode)
+        end
+        c.rules.each do |rule|
+          self.add_rule(rule, c.mode)
+        end
+      end
+      @tokens.each do |token|
+        self.add_rule(token)
+      end
+      @rules.each do |rule|
+        self.add_rule(rule)
+      end
+
+      @tokens = nil
+      @rules = nil
+
+      self
+    end
+
+    def add_rule(r, m = :default)
+      return if r.nil?
+      mode = ((r.mode.nil? || r.mode.to_sym == :default) ? m : r.mode).to_sym
+      @modes[mode] ||= { }
+      @modes[mode][r.name.to_sym] = r
+    end
+
+    def get_rule(m, nom)
+      @modes[m.to_sym].nil? ? nil : @modes[m.to_sym][nom.to_sym]
+    end
+
+    def parse(ctx, nom, s)
+      cursor = Fabulator::Grammar::Cursor.new(self, ctx, s)
+      cursor.anchored = true
+      ret = do_parse(nom, cursor)
+      cursor.do_skip
+      cursor.eof ? ret : nil
+    end
+
+    def match(ctx, nom, s)
+      cursor = Fabulator::Grammar::Cursor.new(self, ctx, s)
+      !do_parse(nom, cursor).nil?
+    end
+
+      protected
+
+    def do_parse(nom, cursor)
+      obj = get_rule(:default, nom)
+      return nil if obj.nil?
+
+      obj.parse(cursor)
+    end
+
+      end
+    end
+  end
+end