sexpistol 0.0.1 → 0.0.2

data/.gitignore

@@ -0,0 +1 @@
+*.gem

data/README.rdoc

@@ -5,19 +5,21 @@ Sexpistol is a simple library for parsing S-Expressions in Ruby. Sexpistol takes
 === Example
 
   (define test (lambda () (
-    (print "blah")
+    (print "Hello world!\n")
     (print 1)
     (print 9.01)
+    (print 2.0e10)
     (print (+ 10 12 13))
   )))
   
 would be parsed by Sexpistol like so:
 
   [:define, :test, [:lambda, [], [
-    [:print, "blah"],
+    [:print, "Hello world!\n"],
     [:print, 1],
     [:print, 9.01],
-    [:print, [:"+", 10, 12, 13]]
+    [:print, 2.0e10],
+    [:print, [:+, 10, 12, 13]]
   ]]]
   
 === Type mappings
@@ -26,9 +28,9 @@ Sexpistol supports all of the standard datatypes and converts them directly to t
 
 * Lists (a b c)
 * Integers (1 2 3)
-* Floats (1.0 2.0 42.9)
+* Floats (1.0 2.0 42.9 3e6 1.2e2)
 * Strings ("\t\"Hello world!\"\n")
-* Symbols (symbol Symbol __symbol__ symbo_l symbol? symbol!)
+* Symbols (symbol Symbol ____symbol____ symbo_l symbol? symbol! + - / ++ a+ e$, etc...)
 
 Sexpistol also supports mapping the Ruby keyword literals (nil, true, false) to their native Ruby types, although this is disabled by default for compatibility. To enable it use `@parser.ruby_keyword_literals = true`, eg:
   

data/VERSION

@@ -1 +1 @@
-0.0.1
+0.0.2

data/lib/sexpistol/sexpistol.rb

@@ -12,9 +12,7 @@ class Sexpistol
   # Parse a string containing an S-Expression into a
   # nested set of Ruby arrays
   def parse_string( string )
-    string.gsub!("(", " ( ")
-    string.gsub!(")", " ) ")
-    string_array = string.split
+    string_array = split_outside_strings(string)
     tokens = process_tokens( string_array )
     check_tokens( tokens )
     structure( tokens )
@@ -24,7 +22,7 @@ class Sexpistol
   # of open and closing parentheses match
   def check_tokens( tokens )
     unless( (tokens.reject {|x| x == "("}).length == (tokens.reject {|x| x == ")"}).length)
-      raise Exception, "Invalid S-Expression. The number of opening and closing parentheses do not match."
+      raise Exception, "Invalid S-Expression. The number of opening and closing parentheses does not match."
     end
   end
 
@@ -41,7 +39,7 @@ class Sexpistol
       tokens << t and next if(is_paren?(t))
       tokens << t.to_f and next if( is_float?(t))
       tokens << t.to_i and next if( is_integer?(t))
-      tokens << t.to_sym and next if( is_identifier?(t) || is_symbol?(t))
+      tokens << t.to_sym and next if( is_symbol?(t))
       tokens << eval(t) and next if( is_string_literal?(t))
       raise "\nUnrecognized token: #{t}\n"
     end
@@ -69,6 +67,31 @@ class Sexpistol
       return program
     end
   end
+  
+  # Split up a string into an array where delimited by whitespace,
+  # except inside string literals
+  def split_outside_strings( string )
+    string_literal_pattern = /"([^"\\]|\\.)*"/
+    string_token = "__++STRING_LITERAL++__"
+    # Find and extract all the string literals
+    string_literals = []
+    string.gsub(string_literal_pattern) {|x| string_literals << x}
+    # Replace all the string literals with a special token
+    string = string.gsub(string_literal_pattern, string_token)
+    # Split the string up on whitespace and parentheses
+    string.gsub!("(", " ( ")
+    string.gsub!(")", " ) ")
+    array = string.split(" ")
+    # replace the special string token with the original string literals
+    array.collect! do |x|
+      if( x == string_token)
+        string_literals.shift
+      else
+        x
+      end
+    end
+    return array
+  end
 
   # Test to see whether or not a string represents the 'nil' literal
   def is_nil?( string )
@@ -97,22 +120,17 @@ class Sexpistol
   
   # Test to see whether or not a string represents a float
   def is_float?( string )
-    is_match?( string, /[\-\+]?[0-9]+\.[0-9]+/ )
+    is_match?( string, /[\-\+]?[0-9]+\.[0-9]+(e[0-9]+)?/ )
   end
 
-  # Test to see whether or not a string represents an identifier
-  def is_identifier?( string )
-    is_match?( string, /_*[a-zA-Z]+[a-zA-Z0-9_]*\??\!?/ )
+  # Test to see whether or not a string represents a symbol
+  def is_symbol?( string )
+    is_match?( string, /[^\"\'\,\(\)]+/ )
   end
   
   # Test to see whether or not a string represents a string literal
   def is_string_literal?( string )
-    is_match?( string, /".*"/)
-  end
-
-  # Test to see whether or not a string represents a symbol
-  def is_symbol?( string )
-    is_match?( string, /[\!\*\^=\/\+\-]+/ )
+    is_match?( string, /"([^"\\]|\\.)*"/)
   end
   
   # Convert a set of nested arrays back into an S-Expression

data/sexpistol.gemspec

@@ -0,0 +1,65 @@
+# Generated by jeweler
+# DO NOT EDIT THIS FILE DIRECTLY
+# Instead, edit Jeweler::Tasks in Rakefile, and run the gemspec command
+# -*- encoding: utf-8 -*-
+
+Gem::Specification.new do |s|
+  s.name = %q{sexpistol}
+  s.version = "0.0.2"
+
+  s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
+  s.authors = ["Aaron Gough"]
+  s.date = %q{2010-10-03}
+  s.description = %q{Sexpistol is an easy-to-use S-Expression parser for Ruby. It is fast and has no dependencies.}
+  s.email = %q{aaron@aarongough.com}
+  s.extra_rdoc_files = [
+    "MIT-LICENSE",
+     "README.rdoc"
+  ]
+  s.files = [
+    ".gitignore",
+     "MIT-LICENSE",
+     "README.rdoc",
+     "Rakefile",
+     "VERSION",
+     "lib/sexpistol.rb",
+     "lib/sexpistol/sexpistol.rb",
+     "sexpistol.gemspec",
+     "test/setup/test_unit_extensions.rb",
+     "test/test_helper.rb",
+     "test/unit/float_literal_test.rb",
+     "test/unit/integer_literal_test.rb",
+     "test/unit/ruby_keyword_literals_test.rb",
+     "test/unit/string_literal_test.rb",
+     "test/unit/structure_test.rb",
+     "test/unit/symbol_test.rb",
+     "test/unit/to_sexp_test.rb"
+  ]
+  s.homepage = %q{http://github.com/aarongough/sexpistol}
+  s.rdoc_options = ["--charset=UTF-8", "--line-numbers", "--inline-source"]
+  s.require_paths = ["lib"]
+  s.rubygems_version = %q{1.3.7}
+  s.summary = %q{An S-Expression Parser Library for Ruby}
+  s.test_files = [
+    "test/setup/test_unit_extensions.rb",
+     "test/test_helper.rb",
+     "test/unit/float_literal_test.rb",
+     "test/unit/integer_literal_test.rb",
+     "test/unit/ruby_keyword_literals_test.rb",
+     "test/unit/string_literal_test.rb",
+     "test/unit/structure_test.rb",
+     "test/unit/symbol_test.rb",
+     "test/unit/to_sexp_test.rb"
+  ]
+
+  if s.respond_to? :specification_version then
+    current_version = Gem::Specification::CURRENT_SPECIFICATION_VERSION
+    s.specification_version = 3
+
+    if Gem::Version.new(Gem::VERSION) >= Gem::Version.new('1.2.0') then
+    else
+    end
+  else
+  end
+end
+

data/test/unit/float_literal_test.rb

@@ -25,6 +25,11 @@ class FloatLiteralTest < Test::Unit::TestCase
     ast = @parser.parse_string("1.0000127829")
     assert_equal [1.0000127829], ast
   end
+  
+  test "should parse sexp containing a float defined in scientific notation" do
+    ast = @parser.parse_string("1.0e6")
+    assert_equal [1.0e6], ast
+  end
     
     
 end

data/test/unit/string_literal_test.rb

@@ -25,5 +25,15 @@ class StringLiteralTest < Test::Unit::TestCase
     ast = @parser.parse_string('"\n\t\r"')
     assert_equal ["\n\t\r"], ast
   end
+  
+  test "should parse string literal containing spaces" do
+    ast = @parser.parse_string('"blah foo"')
+    assert_equal ["blah foo"], ast
+  end
+  
+  test "should parse string literal containing newlines" do
+    ast = @parser.parse_string('"blah' + "\n" + 'foo"')
+    assert_equal ["blah\nfoo"], ast
+  end
     
 end

data/test/unit/structure_test.rb

@@ -11,6 +11,11 @@ class RubyKeywordLiteralsTest < Test::Unit::TestCase
     assert_equal [[:this, [:is, [:an, [:s_expression]]]]], ast
   end
   
+  test "should create nested set of arrays from s-expression with string literals" do
+    ast = @parser.parse_string('(this (is (an ("s_expression"))))')
+    assert_equal [[:this, [:is, [:an, ["s_expression"]]]]], ast
+  end
+  
   test "should raise error on broken s-expression" do
     assert_raises Exception do
       ast = @parser.parse_string('(this (is (an (s_expression) too)')

data/test/unit/symbol_test.rb

@@ -45,5 +45,30 @@ class SymbolTest < Test::Unit::TestCase
     ast = @parser.parse_string("__TestSymbol_TEST__?")
     assert_equal [:__TestSymbol_TEST__?], ast
   end
+  
+  test "should parse symbol containing addition operators" do
+    ast = @parser.parse_string("+")
+    assert_equal [:+], ast
+  end
+  
+  test "should parse symbol containing multiplication operators" do
+    ast = @parser.parse_string("*")
+    assert_equal [:*], ast
+  end
+  
+  test "should parse symbol containing subtraction operators" do
+    ast = @parser.parse_string("-")
+    assert_equal [:-], ast
+  end
+  
+  test "should parse symbol containing division operators" do
+    ast = @parser.parse_string("/")
+    assert_equal [:"/"], ast
+  end
+  
+  test "should parse symbol containing any character except single and double quotes, backquote, parentheses and comma" do
+    ast = @parser.parse_string("~1!2@3#4$%5^6&7*890-_+=|\]}[{poiuytrewqasdfghjklmnbvcxzZXCVBNMLKJHGFDSAQWERTYUIOP:;/?><")
+    assert_equal [:"~1!2@3#4$%5^6&7*890-_+=|\]}[{poiuytrewqasdfghjklmnbvcxzZXCVBNMLKJHGFDSAQWERTYUIOP:;/?><"], ast
+  end
     
 end

metadata

@@ -5,8 +5,8 @@ version: !ruby/object:Gem::Version
   segments: 
   - 0
   - 0
-  - 1
-  version: 0.0.1
+  - 2
+  version: 0.0.2
 platform: ruby
 authors: 
 - Aaron Gough
@@ -14,7 +14,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2010-10-02 00:00:00 -04:00
+date: 2010-10-03 00:00:00 -04:00
 default_executable: 
 dependencies: []
 
@@ -28,12 +28,14 @@ extra_rdoc_files:
 - MIT-LICENSE
 - README.rdoc
 files: 
+- .gitignore
 - MIT-LICENSE
 - README.rdoc
 - Rakefile
 - VERSION
 - lib/sexpistol.rb
 - lib/sexpistol/sexpistol.rb
+- sexpistol.gemspec
 - test/setup/test_unit_extensions.rb
 - test/test_helper.rb
 - test/unit/float_literal_test.rb