sexpr 0.3.0 → 0.4.0

data/CHANGELOG.md

@@ -1,3 +1,32 @@
+# 0.4.0 / 2012-02-23
+
+* Major enhancements
+
+  * A processing/rewriting framework has been added to Sexpr. See the `Processor` and `Rewriter`
+    classes, as well as the boolean expression example.
+  * Tracking markers can now decorate s-expressions, provided they include the `Sexpr` module.
+    Tracking markers are a simple Hash of meta-information (i.e. not taken into account for
+    equality for s-expressions). Such markers can be set with `Grammar#sexpr(sexpr, markers)`.
+    Default markers are typically provided by parsers for traceability of the s-expression
+    with the source text it comes from.
+
+* Minor enhancements
+
+  * `Citrus::Parser#parse` is now idempotent and so is `Grammar#parse` therefore.
+  * The module to use for finding tag modules through `const_get` can now be overridden in
+    `Grammar#tagging_reference`.
+  * Default parsing options can now be specified in `Grammar#default_parse_options`. These
+    options are used by `Grammar#sexpr` when parsing is needed.
+
+* Breaking changes
+
+  * `Parser.factor` does no longer accept options. This is to avoid the 'yet another options'
+    symptom and favor convention over configuration.
+  * Accordingly, `Sexpr::Citrus::Parser` no longer takes options at construction either.
+  * `Grammar#sexpr` does no longer allow parsing options as second argument, but takes tracking
+    markers (see enhancements). To palliate to this, default parsing options can now be
+    specified through `Grammar#default_parse_options` (see enhancements).
+
 # 0.3.0 / 2012-02-21
 
 * Breaking changes

data/README.md

@@ -2,6 +2,18 @@
 
 A ruby compilation framework around s-expressions.
 
+## Links
+
+https://github.com/blambeau/sexpr
+
+## Features/Problems
+
+* Provides a YAML format for describing grammars (abstract syntax trees, more precisely).
+* Provides a simple way to check the validity of a s-expression against a given grammar.
+* Provides a framework for processing and rewriting abstract syntax trees.
+* Focusses on the semantic pass, not the syntactic one.
+* Smoothly, yet not tightly, integrates with the Citrus PEG parser (for the syntactic pass).
+
 ## Example
 
     # Let load a grammar defined in YAML
@@ -50,12 +62,26 @@ A ruby compilation framework around s-expressions.
     # such s-expressions
     expr = grammar.sexpr([:bool_lit, true])
 
+    Sexpr === expr
+    # => true
+
     expr.sexpr_type
     # => :bool_lit
 
     expr.sexpr_body
     # => [true]
 
-## Links
+    # Rewriting s-expressions through copying is easy...
+    copy = expr.sexpr_copy do |base,child|
+      # copy a s-expression ala Enumerable#inject (base is [:bool_lit] initially)
+      base << [:bool_lit, !child]
+    end
+    # => [:bool_lit, [:bool_lit, false]]
+
+    # ... and is tag preserving (including User-included modules)
+    Sexpr === copy
+    # true
+
+### Where to read next?
 
-https://github.com/blambeau/sexpr
+Have a look at the examples directory.

data/examples/bool_expr/bool_expr.citrus

@@ -16,21 +16,21 @@ grammar BoolExpr::Parser
 
   rule bool_or
       (l:bool_and spaces 'or' spaces r:bool_or){
-        [:bool_or, l.value, r.value]
+        [:bool_or, l.sexpr, r.sexpr]
       }
     | bool_and
   end
 
   rule bool_and
       (l:bool_not spaces 'and' spaces r:bool_and){
-        [:bool_and, l.value, r.value]
+        [:bool_and, l.sexpr, r.sexpr]
       }
     | bool_not
   end
 
   rule bool_not
       ('not' spacing e:bool_not){
-        [:bool_not, e.value]
+        [:bool_not, e.sexpr]
       }
     | bool_term
   end
@@ -41,7 +41,7 @@ grammar BoolExpr::Parser
 
   rule bool_paren
     ('(' spacing e:bool_or spacing ')'){
-      e.value
+      e.sexpr
     }
   end
 
@@ -52,13 +52,13 @@ grammar BoolExpr::Parser
   end
 
   rule var_ref
-    (!(keyword spacing) [a-z]+){
+    (!(keyword (spaces | !.)) [a-z]+){
       [:var_ref, strip]
     }
   end
 
   rule spacing
-    [ \t]+ | &'(' | !.
+    [ \t]*
   end
 
   rule spaces

data/examples/bool_expr/bool_expr.rb

@@ -1,4 +1,5 @@
 require 'sexpr'
+require 'citrus'
 
 # Let load the grammar from the .yml definition file.
 BoolExpr = Sexpr.load File.expand_path('../bool_expr.sexp.yml', __FILE__)
@@ -28,7 +29,34 @@ module BoolExpr
     (rule.to_s =~ /^bool_(.*)$/) ? const_get($1.to_sym) : rule
   end
 
-end
+  # This class pushes `[:not, ...]` as far as possible in boolean expressions.
+  # It provides an example of s-expression rewriter
+  class NotPushProcessor < Sexpr::Rewriter
+
+    # Let the default implementation know that we are working on the BoolExpr
+    # grammar. This way, all rewriting results will automatically be tagged
+    # with the correct modules above (And, Not, ...)
+    grammar BoolExpr
+
+    # The main rewriting rule, that pushes a NOT according to the different
+    # cases
+    def on_bool_not(sexpr)
+      case expr = sexpr.last
+      when And then call [:bool_or,  [:bool_not, expr[1]], [:bool_not, expr[2]] ]
+      when Or  then call [:bool_and, [:bool_not, expr[1]], [:bool_not, expr[2]] ]
+      when Not then call expr.last
+      when Lit then [:bool_lit, !expr.last]
+      else
+        sexpr
+      end
+    end
+
+    # By default, we simply copy the node and apply rewriting rules on children
+    alias :on_missing :copy_and_apply
+
+  end # class NotPushProcessor
+
+end # module BoolExpr
 
 describe BoolExpr do
   subject{ BoolExpr }
@@ -37,6 +65,8 @@ describe BoolExpr do
 
     it 'parses boolean expressions without error' do
       subject.parse("x and y").should be_a(Citrus::Match)
+      subject.parse("not(y)").should be_a(Citrus::Match)
+      subject.parse("not(true)").should be_a(Citrus::Match)
     end
 
     it 'provides a shortcut to get s-expressions directly' do
@@ -83,4 +113,41 @@ describe BoolExpr do
 
   end # validating
 
+  describe BoolExpr::NotPushProcessor do
+
+    def _(expr)
+      BoolExpr.sexpr(expr)
+    end
+
+    def rw(expr)
+      BoolExpr::NotPushProcessor.new.call(expr)
+    end
+
+    it 'does nothing on variable references' do
+      rw("not x").should eq([:bool_not, [:var_ref, "x"]])
+    end
+
+    it 'rewrites literals through negating them' do
+      rw("not true").should  eq(_ "false")
+      rw("not false").should eq(_ "true")
+    end
+
+    it 'rewrites not through removing them' do
+      rw("not not true").should eq(_ "true")
+    end
+
+    it 'rewrites or through and of negated terms' do
+      rw("not(x or y)").should eq(_ "not(x) and not(y)")
+    end
+
+    it 'rewrites and through or of negated terms' do
+      rw("not(x and y)").should eq(_ "not(x) or not(y)")
+    end
+
+    it 'rewrites recursively' do
+      rw("not(x and not(y))").should eq(_ "not(x) or y")
+    end
+
+  end # rewriting
+
 end if defined?(RSpec)

data/lib/sexpr.rb

@@ -1,38 +1,45 @@
+require 'yaml'
 require_relative "sexpr/version"
 require_relative "sexpr/loader"
 require_relative "sexpr/errors"
+require_relative "sexpr/node"
+require_relative "sexpr/grammar"
+require_relative "sexpr/matcher"
+require_relative "sexpr/parser"
+require_relative "sexpr/processor"
+require_relative "sexpr/rewriter"
 #
 # A helper to manipulate sexp grammars
 #
 module Sexpr
+  extend Grammar::Tagging
 
   PathLike = lambda{|x|
     x.respond_to?(:to_path) or (x.is_a?(String) and File.exists?(x))
   }
 
-  def self.load(input)
-    defn = case input
-            when PathLike
-              require 'yaml'
-              path = input.respond_to?(:to_path) ? input.to_path : input.to_s
-              YAML.load_file(path).merge(:path => input)
-            when String
-              require 'yaml'
-              YAML.load(input)
-            when Hash
-              input
-            else
-              raise ArgumentError, "Invalid argument for Sexpr::Grammar: #{input}"
-            end
-    Grammar.new defn
+  def self.load(input, options = {})
+    case input
+    when PathLike then load_file   input, options
+    when String   then load_string input, options
+    when Hash     then load_hash   input, options
+    else
+      raise ArgumentError, "Invalid argument for Sexpr::Grammar: #{input}"
+    end
   end
 
-  def self.sexpr(arg)
-    Object.new.extend(Sexpr::Grammar::Tagging).sexpr(arg)
+  def self.load_file(input, options = {})
+    path = input.to_path rescue input.to_s
+    load_hash YAML.load_file(path), options.merge(:path => input)
   end
 
-end # module Sexpr
-require_relative "sexpr/node"
-require_relative "sexpr/grammar"
-require_relative "sexpr/matcher"
-require_relative "sexpr/parser"
+  def self.load_string(input, options = {})
+    load_hash YAML.load(input), options
+  end
+
+  def self.load_hash(input, options = {})
+    raise ArgumentError, "Invalid grammar definition: #{input}" unless Hash===input
+    Grammar.new input, options
+  end
+
+end # module Sexpr

data/lib/sexpr/errors.rb

@@ -12,4 +12,7 @@ module Sexpr
   class NoParserError < Error
   end
 
+  class UnexpectedSexprError < Error
+  end
+
 end

data/lib/sexpr/grammar.rb

@@ -6,21 +6,22 @@ module Sexpr
   module Grammar
     include Options
     include Matching
-    include Parsing
     include Tagging
+    include Parsing
 
-    def self.new(options = {})
-      unless options.is_a?(Hash)
-        raise ArgumentError, "Invalid grammar definition: #{options.inspect}"
-      end
+    def self.new(input = {}, options = {})
       Module.new.tap{|g|
         g.instance_eval{
           include(Grammar)
           extend(self)
-          install_options(options)
+          install_options(input.merge(options))
         }
       }
     end
 
+    def tagging_reference
+      self
+    end
+
   end # module Grammar
 end # module Sexpr

data/lib/sexpr/grammar/options.rb

@@ -39,6 +39,9 @@ module Sexpr
       def install_parser
         @parser = option(:parser)
         if @parser.is_a?(String) && !File.exists?(@parser)
+          unless path
+            raise Errno::ENOENT, "#{@parser} (no main path)"
+          end
           @parser = File.join(File.dirname(path), @parser)
         end
         @parser = Parser.factor(@parser) if @parser

data/lib/sexpr/grammar/parsing.rb

@@ -2,7 +2,12 @@ module Sexpr
   module Grammar
     module Parsing
 
-      def parse(input, options = {})
+      def default_parse_options
+        {}
+      end
+
+      def parse(input, options = nil)
+        options = default_parse_options.merge(options || {})
         parser!.parse(input, options)
       end
 

data/lib/sexpr/grammar/tagging.rb

@@ -11,39 +11,58 @@ module Sexpr
         mod.to_s.gsub(/[A-Z]/){|x| "_#{x.downcase}"}[1..-1].to_sym
       end
 
-      def sexpr(input, options = {})
+      def tagging_reference
+        nil
+      end
+
+      def sexpr(input, markers = nil)
         case input
         when Array
-          tag_sexpr input
+          tag_sexpr input, tagging_reference, markers
         else
-          tag_sexpr parser!.sexpr(input, options)
+          sexpr = parser!.to_sexpr(parse(input))
+          tag_sexpr sexpr, tagging_reference, markers, true
         end
       end
 
       private
 
-      def tag_sexpr(sexpr, reference = self)
-        if looks_a_sexpr?(sexpr)
-          sexpr = tag_sexpr_with_user_module(sexpr, reference)
-          sexpr[1..-1].each do |child|
-            tag_sexpr(child, reference)
-          end
+      def tag_sexpr(sexpr, reference, markers = nil, force = false)
+        return sexpr unless looks_a_sexpr?(sexpr)
+        return sexpr if Sexpr===sexpr and not(force) and markers.nil?
+
+        # set the Sexpr modules
+        sexpr.extend(Sexpr) unless Sexpr===sexpr
+        tag_sexpr_with_user_module(sexpr, reference) if reference
+
+        # set the markers if any
+        if markers
+          markers = sexpr.tracking_markers.merge(markers) if Sexpr===sexpr
+          sexpr.tracking_markers = markers
+        end
+
+        # recurse
+        sexpr[1..-1].each do |child|
+          tag_sexpr(child, reference, nil, force)
         end
         sexpr
       end
 
       def tag_sexpr_with_user_module(sexpr, reference)
-        sexpr.extend(Sexpr)
         rulename = sexpr.first
         modname  = rule2modname(rulename)
         mod      = reference.const_get(modname) rescue nil
-        mod ? sexpr.extend(mod) : sexpr
+        sexpr.extend(mod) if mod
       end
 
       def looks_a_sexpr?(arg)
         arg.is_a?(Array) and arg.first.is_a?(Symbol)
       end
 
+      def parser!
+        raise NoParserError, "No parser set.", caller
+      end
+
     end # module Tagging
   end # module Grammar
 end # module Sexpr

data/lib/sexpr/node.rb

@@ -1,6 +1,16 @@
 module Sexpr
   module Node
 
+    EMPTY_TRACKING_MARKERS = {}
+
+    def tracking_markers
+      @tracking_markers ||= EMPTY_TRACKING_MARKERS
+    end
+
+    def tracking_markers=(markers)
+      @tracking_markers = markers
+    end
+
     def sexpr_type
       first
     end
@@ -11,6 +21,26 @@ module Sexpr
     end
     alias :sexp_body :sexpr_body
 
+    def sexpr_copy(&block)
+      if block
+        copy = sexpr_copy_tagging([ sexpr_type ])
+        sexpr_body.inject(copy, &block)
+      else
+        sexpr_copy_tagging(self[0..-1])
+      end
+    end
+    alias :dup :sexpr_copy
+
+    private
+
+    def sexpr_copy_tagging(copy)
+      (class << self; self; end).included_modules.each do |mod|
+        copy.extend(mod) unless mod === copy
+      end
+      copy.tracking_markers = tracking_markers
+      copy
+    end
+
   end # module Node
   include Node
 end # module Sexpr