sexp_path 0.4.0

data/README.rdoc

@@ -0,0 +1,167 @@
+= SexpPath
+
+Structural pattern matching against S-Expressions.
+
+SexpPath allows you to define patterns that can be matched against S-Expressions.
+SexpPath draws inspiration from Regular Expressions, XPath, and CSS Selectors.
+
+I'm still figuring out how SexpPath should work so either fork this, or send me
+some feedback.
+  http://github.com/adamsanderson/sexp_path
+  netghost@gmail.com
+
+== Installation
+
+SexpPath is distributed as a ruby gem:
+    
+    gem install adamsanderson-sexp_path
+
+== Notation
+
+In ruby you're most likely to come across S-Expressions when dealing with
+ParseTree's representation of the abstract syntax tree. An S-Expression is
+just a set of nested lists.  The SexpProcessor library displays them like this:
+
+    s(:a, :b, 
+      s(:c)
+    )
+
+Where that means that there is a list containing `:a`, `:b`, and then another list which
+contains `:c`.  We will refer to `:a`,`:b`, and `:c` as atoms, while 
+`s( something )` is an S-Expression or Sexp.
+
+== General Syntax
+
+SexpPath is an internal ruby DSL, which means a SexpPath query is valid ruby code.  
+SexpPath queries are built with the SexpQueryBuilder through the Q? convenience
+method:
+  
+    Q?{ s(:a, :b, :c)}    # Matches s(:a, :b, :c)
+  
+This will match the S-Expression `s(:a, :b, :c)`.  If you want to match something 
+more complicated, you will probably want to employ one of the many matchers built
+into SexpPath.
+    
+[Wild Card] Matches anything.
+
+    _  => s(), :a, or s(:cat)
+    
+[Atom] Matches any atom (or symbol).
+
+    atom => :a, :b, or :cat
+
+[Pattern] Matches any atom that matches the given string or regular expression.
+
+    m('cat') => :cat
+    m(/rat/) => :rat, :brat, or :rate
+    m(/^test_/) => :test_sexp_path
+    
+[Includes] Matches any S-Expression that includes the sub expression.
+
+    include(:a) => s(:a), s(:a, :b), or s(:cat, :a, :b)
+    include( s(:cat) ) => s(:pet, s(:cat))
+    
+[Child] Matches any S-Expression that has the sub expression as a child.
+
+    child( s(:a) ) => s(:b, s(:a)) or even s(s(s(s(s( s(:a))))))
+    
+[Sibling] Matches any S-Expression that has the second expression as a sibling.
+    s(:a) >> s(:c) => s( s(:a), s(:b), s(:c) )
+
+[Type] The sexp type is considered to be the first atom.  This matches any expression that has the given type.
+
+    type(:a) => s(:a), s(:a, :b), or s(:a, :b, s(:c))
+    
+[Any] Matches any sub expression
+
+    any( s(:a), s(:b) ) => s(:a) or s(:b)
+    any( s(:a), s(atom, :b) ) => s(:a), s(:a, :b), or s(:cat, :b)
+
+[All] Matches anything that satisfies all the sub expressions
+
+    all( s(:a, atom), s(atom, :b) ) => s(:a,:b)
+    
+[Not] Negates a matcher
+
+    -s(:a) => s(:a,:b), s(:b), but not s(:a)
+    s(is_not :a) => s(:b), s(:c), but not s(:a) or s(:a, :b)
+    
+== Searching
+
+You may use any SexpPath to search an S-Expression.  SexpPath defines the `/` operator as search,
+so to search `s( s(:a) )`  for `s(:a)` you may just do:
+    
+    s( s(:a) ) / Q?{ s(:a) }
+    
+This will return a collection with just one result which is `s(:a)`.  You could also do something
+more interesting:
+
+    s( s(:a), s(:b) ) / Q?{ s(atom) }
+    
+This will return two matches which are `s(:a)` and `s(:b)`.  You can also chain searches, so this
+works just fine as well:
+
+    sexp = s(:class, :Calculator,
+      s(:defn, :add),
+      s(:defn, :sub)
+    ) 
+    
+    sexp / Q?{ s(:class, atom, _) } / Q?{ s(:defn, _) }
+    
+In this case you would get back `s(:defn, :add)` and `s(:defn, :sub)`.
+
+== Capturing
+
+It is useful to also capture results from your queries.  So using
+the same Sexp from above we could modify our query to actually capture some names.
+Capturing is done by using `%` operator followed by the name you would like the value
+to be captured as.
+
+    sexp / Q?{ s(:class, atom % 'class_name', _) } / Q?{ s(:defn, _ % 'method_name') }
+  
+The results will now capture `:Calculator` in `class_name`, and then `:add` and `:sub`
+in `method_name`.
+
+== Examples
+
+Here is an example of using SexpPath to grab all the classes and their methods from
+a file:
+  
+    require 'rubygems'
+    require 'sexp_path'
+    require 'parse_tree'
+  
+    path = ARGV.shift
+    code = File.read(path)
+    sexp = Sexp.from_array(ParseTree.new.parse_tree_for_string(code, path))
+
+    class_query = Q?{ s(:class, atom % 'class_name', _, _) }
+    method_query = Q?{ s(:defn, atom % 'method_name', _ ) }
+
+    results = sexp / class_query / method_query
+
+    puts path
+    puts "-" * 80
+
+    results.each do |sexp_result|
+      class_name = sexp_result['class_name']
+      method_name = sexp_result['method_name']
+      puts "#{class_name}##{method_name}"
+    end
+  
+Neat huh?  Check the `examples` folder for some more little apps.
+
+== Project Information
+
+Hop in and fork it or add some issues over at GitHub: 
+  http://github.com/adamsanderson/sexp_path
+
+Ideas for Hacking on SexpPath:
+  
+  * More examples
+  * Add new matchers
+  * Connivence matchers, for instance canned matchers for matching ruby classes, methods, etc
+
+I'd love to see what people do with this library, let me know if you find it useful.
+
+    Adam Sanderson, netghost@gmail.com

data/Rakefile

@@ -0,0 +1,41 @@
+require 'rake'
+require 'rake/testtask'
+require 'rake/rdoctask'
+
+begin
+  require 'jeweler'
+  
+  Jeweler::Tasks.new do |s|
+    s.name = "sexp_path"
+    s.summary = "Pattern matching for S-Expressions (sexp)."
+    s.description = <<-DESC
+      Allows you to do example based pattern matching and queries against S Expressions (sexp).
+    DESC
+    s.email = "netghost@gmail.com"
+    s.homepage = "http://github.com/adamsanderson/sexp_path"
+    s.authors = ["Adam Sanderson"]
+    s.files = FileList["[A-Z]*", "{bin,lib,test,examples}/**/*"]
+    
+    s.add_dependency 'sexp_processor', '~> 3.0'
+    
+    # Testing
+    s.test_files = FileList["test/**/*_test.rb"]
+    s.add_development_dependency 'ParseTree', '~> 2.1'
+  end
+
+rescue LoadError
+  puts "Jeweler not available. Install it for jeweler-related tasks with: sudo gem install technicalpickles-jeweler -s http://gems.github.com"
+end
+
+Rake::RDocTask.new do |t|
+  t.main = "README.rdoc"
+  t.rdoc_files.include("README.rdoc", "lib/**/*.rb")
+end
+
+Rake::TestTask.new do |t|
+  t.libs << 'lib'
+  t.pattern = 'test/**/*_test.rb'
+  t.verbose = false
+end
+
+task :default => :test

data/TODO

@@ -0,0 +1,4 @@
+* Clean up class organization
+* Document extending SexpPath
+* Add examples of extending matchers
+* AssertSanity!

data/VERSION.yml

@@ -0,0 +1,4 @@
+--- 
+:minor: 4
+:patch: 0
+:major: 0

data/examples/print_methods.rb

@@ -0,0 +1,28 @@
+require 'rubygems'
+require File.dirname(__FILE__) + '/../lib/sexp_path'
+require 'parse_tree'
+
+path = ARGV.shift
+if !path
+  puts "Prints classes and methods in a file"
+  puts "usage:"
+  puts "  ruby print_methods.rb <path>"
+  exit
+end
+
+code = File.read(path)
+sexp = Sexp.from_array(ParseTree.new.parse_tree_for_string(code, path))
+
+class_query = Q?{ s(:class, atom % 'class_name', _, _) }
+method_query = Q?{ s(:defn, atom % 'method_name', _ ) }
+
+results = sexp / class_query / method_query
+
+puts path
+puts "-" * 80
+
+results.each do |sexp_result|
+  class_name = sexp_result['class_name']
+  method_name = sexp_result['method_name']
+  puts "#{class_name}##{method_name}"
+end

data/examples/sexp_grep.rb

@@ -0,0 +1,46 @@
+require 'rubygems'
+require File.dirname(__FILE__) + '/../lib/sexp_path'
+require 'parse_tree'
+
+# Example program, this will scan a file for anything
+# matching the Sexp passed in.
+
+
+pattern = ARGV.shift
+paths = ARGV
+
+if paths.empty? || !pattern
+  puts "Prints classes and methods in a file"
+  puts "usage:"
+  puts "  ruby sexp_grep.rb <pattern> <path>"
+  puts "example:"
+  puts "  ruby sexp_grep.rb t(:defn) *.rb"
+  exit
+end
+
+begin
+  # Generate the pattern, we use a little instance_eval trickery here. 
+  pattern = SexpPath::SexpQueryBuilder.instance_eval(pattern)
+rescue Exception=>ex
+  puts "Invalid Pattern: '#{pattern}'"
+  puts "Trace:"
+  puts ex
+  puts ex.backtrace
+  exit 1
+end
+
+# For each path the user defined, search for the SexpPath pattern
+paths.each do |path|  
+  # Parse it with ParseTree, and append line numbers
+  sexp = sexp = LineNumberingProcessor.process_file(path)
+  found = false
+  
+  # Search it with the given pattern, printing any results
+  sexp.search_each(pattern) do |match|
+    if !found
+      puts path
+      found = true
+    end
+    puts "%4i: %s" % [match.sexp.line, match.sexp.inspect]
+  end
+end

data/lib/sexp_path.rb

@@ -0,0 +1,57 @@
+require 'rubygems'
+require 'sexp_processor'
+
+module SexpPath
+  
+  # SexpPath Matchers are used to build SexpPath queries.
+  #
+  # See also: SexpQueryBuilder
+  module Matcher  
+  end
+end
+
+sexp_path_root = File.dirname(__FILE__)+'/sexp_path/'
+%w[
+  traverse 
+  sexp_query_builder
+  sexp_result
+  sexp_collection
+  
+  line_numbering_processor
+  
+  matcher/base
+  matcher/any
+  matcher/all
+  matcher/not
+  matcher/child
+  matcher/block
+  matcher/atom
+  matcher/pattern
+  matcher/type
+  matcher/wild
+  matcher/include
+  matcher/sibling
+  
+].each do |path|
+  require sexp_path_root+path
+end
+
+# Pattern building helper, see SexpQueryBuilder
+def Q?(&block)
+  SexpPath::SexpQueryBuilder.do(&block)
+end
+
+# SexpPath extends Sexp with Traverse.
+# This adds support for searching S-Expressions
+class Sexp
+  include SexpPath::Traverse
+  
+  # Extends Sexp to allow any Sexp to be used as a SexpPath matcher
+  def satisfy?(o, data={})
+    return false unless o.is_a? Sexp
+    return false unless length == o.length
+    each_with_index{|c,i| return false unless c.is_a?(Sexp) ? c.satisfy?( o[i], data ) : c == o[i] }
+
+    capture_match(o, data)
+  end
+end

data/lib/sexp_path/line_numbering_processor.rb

@@ -0,0 +1,60 @@
+require 'parse_tree' rescue nil
+
+# Transforms a Sexp, keeping track of newlines.  This uses the internal ruby newline nodes
+# so they must be included in the Sexp to be transformed.  If ParseTree is being used, it should
+# be configured to include newlines:
+#
+#   parser = ParseTree.new(true) # true => include_newlines
+# 
+# LineNumberingProcessor.rewrite_file(path) should be used as a short cut if ParseTree is available.
+#
+class LineNumberingProcessor < SexpProcessor
+  # Helper method for generating a Sexp with line numbers from a file at +path+.
+  #
+  # Only available if ParseTree is loaded.
+  def self.rewrite_file(path)
+    raise 'ParseTree must be installed.' unless Object.const_defined? :ParseTree
+    
+    code = File.read(path)
+    sexp = Sexp.from_array(ParseTree.new(true).parse_tree_for_string(code, path).first)
+    processor = LineNumberingProcessor.new
+    
+    # Fill in the first lines with a value
+    sexp.line = 0
+    sexp.file = path
+    
+    # Rewrite the sexp so that everything gets a line number if possible.
+    processor.rewrite sexp
+  end
+  
+  # Creates a new LineNumberingProcessor.
+  def initialize()
+    super
+    @unsupported.delete :newline
+  end
+  
+  # Rewrites a Sexp using :newline nodes to fill in line and file information.
+  def rewrite exp
+    unless exp.nil?
+      if exp.sexp_type == :newline
+        @line = exp[1]
+        @file = exp[2]
+      end
+      
+      exp.file ||= @file
+      exp.line ||= @line
+    end
+    
+    super exp
+  end
+  
+  private
+  # Removes newlines from the expression, they are read inside of rewrite, and used to give
+  # the other nodes a line number and file.
+  def rewrite_newline(exp)
+    # New lines look like:
+    #  s(:newline, 21, "test/sample.rb", s(:call, nil, :private, s(:arglist)) )
+    sexp = exp[3]
+    rewrite(sexp)
+  end
+end

data/lib/sexp_path/matcher/all.rb

@@ -0,0 +1,20 @@
+# See SexpQueryBuilder.all
+class SexpPath::Matcher::All < SexpPath::Matcher::Base
+  attr_reader :options
+  
+  # Create an All matcher which will match all of the +options+.
+  def initialize(*options)
+    @options = options
+  end
+  
+  # Satisfied when all sub expressions match +o+
+  def satisfy?(o, data={})
+    return nil unless options.all?{|exp| exp.is_a?(Sexp) ? exp.satisfy?(o, data) : exp == o}
+  
+    capture_match o, data
+  end
+
+  def inspect
+    options.map{|o| o.inspect}.join(' & ')
+  end
+end

data/lib/sexp_path/matcher/any.rb

@@ -0,0 +1,20 @@
+# See SexpQueryBuilder.any
+class SexpPath::Matcher::Any < SexpPath::Matcher::Base
+  attr_reader :options
+  
+  # Create an Any matcher which will match any of the +options+.
+  def initialize(*options)
+    @options = options
+  end
+
+  # Satisfied when any sub expressions match +o+
+  def satisfy?(o, data={})
+    return nil unless options.any?{|exp| exp.is_a?(Sexp) ? exp.satisfy?(o, data) : exp == o}
+  
+    capture_match o, data
+  end
+  
+  def inspect
+    options.map{|o| o.inspect}.join(' | ')
+  end
+end