lxl 0.3.8 → 0.4.0

data/CHANGES

@@ -1,3 +1,11 @@
+0.4.0
+
+- Tokenizing greatly refactored
+  - All tokenizing code moved into tokenize (out of eval)
+  - Token class added
+  - Fixes bug with text (non-formula) strings and constant parsing
+- Removed unnessecary Lexer code
+
 0.3.8
 
 - Range#first_colnum and Range#last_colnum added (both use LXL#xlcolnum)

data/VERSION

@@ -1 +1 @@
-0.3.8
+0.4.0

data/lib/lxl.rb

@@ -35,7 +35,7 @@
 #   NOW      ()      # current date/time value
 #   DATE     (y,m,d) # date value
 #   TIME     (h,m,s) # time value
-#   DATETIME (text)  # convert a date/time string into a date/time value
+#   DATETIME (string)  # convert a date/time string into a date/time value
 # 
 # =List Functions
 # 
@@ -76,7 +76,7 @@
 # 
 #   class MyLXLNamespace < LXL::Namespace
 #     NAME = 'John Doe'
-#     def upper(text) text.to_s.upcase end
+#     def upper(string) string.to_s.upcase end
 #   end
 # 
 #   class MyLXL < LXL::Parser
@@ -86,7 +86,7 @@
 #   MyLXL.eval('=UPPER(NAME)')
 #   # => JOHN DOE
 # 
-# =Symbol registration
+# =Symbol Registration
 # 
 # Register uppercase constants of the same name and value.
 # 
@@ -96,7 +96,7 @@
 #   LXL.new(MyNamespace.new).eval('=LIST(FOO, BAR)')
 #   # => [:FOO, :BAR]
 #
-# =Deferred function calls
+# =Deferred Calls
 # 
 # LXL::Deferred snapshots the symbol/arguments of a function call for later use.
 # 
@@ -154,7 +154,7 @@ module LXL
   def xlcolnum(colname)
     count = 0
     letters = colname.to_s.split(//)
-    map = Proc.new { |c| ::Range.new('A','Z').collect.index(c)+1 }
+    map = proc { |c| ('A'..'Z').collect.index(c)+1 }
     letters[0..-2].each { |l| count += 26*map.call(l) }
     count += map.call(letters.last)
     count
@@ -166,6 +166,7 @@ end
 #
 #  ;  Statement separator
 #  ,  Argument separator
+#  =  Formula prefix
 #  (  Tuple open
 #  )  Tuple close
 #  w  Whitespace
@@ -175,13 +176,21 @@ end
 #  p  Percentage
 #  f  Float
 #  i  Integer
-#  t  Token
+#  u  Unknown
 #
 #  F Function
 #  C Constant
 #
 class LXL::Parser
 
+  class Token
+    attr_accessor :type, :value
+    def initialize(type, value) @type = type; @value = value end
+    def whitespace?() value.to_s.strip.empty? end
+    def to_s() "T:#{@type.chr}:#{@value}" end
+    alias inspect to_s
+  end
+
   RUBY_OPERATORS  = ['+', '-', '*', '/', '<=', '>=', '==', '!=', '<', '>', '+', '**']
   EXCEL_OPERATORS = ['+', '-', '*', '/', '<=', '>=', '=',  '<>', '<', '>', '&', '^' ]
 
@@ -218,6 +227,7 @@ class LXL::Parser
     @lexer = self.class.lexer_class.new([
       [/\A;+/, ?;],              # Statement separator
       [/\A,+/, ?,],              # Argument separator
+      [/\A`=/, ?=],              # Formula prefix
       [/\A\(/, ?(],              # Tuple open
       [/\A\)/, ?)],              # Tuple close
       [/\A\s+/, ?w],             # Whitespace
@@ -227,29 +237,15 @@ class LXL::Parser
       [/\A\d+(\.\d+)?%/, ?p],    # Percentage
       [/\A\d+\.\d+/, ?f],        # Float
       [/\A\d+/, ?i],             # Integer
-      [/\A\w+/, ?t],             # Token
-    ], false)
+      [/\A\w+/, ?u],             # Unknown
+    ])
   end
 
   # Evaluate a formula.
   #
-  def eval(formula)
-    formulas = Array.new << Array.new
-    formula = formula.to_s.gsub(/(\n)(\s*=)/,'\1;\2') # infer statement separators
-    types,tokens = tokenize(formula)
-    tokens.each_index { |i| tokens[i] == ';' ? formulas << Array.new : formulas.last << [types[i], tokens[i]] }
-    formulas.collect! { |f|
-      types  = f.collect { |t| t.first }
-      tokens = f.collect { |t| t.last  }
-      token = tokens.join.strip
-      if token =~ /\A=/
-        tokens.each_index { |i| tokens[i] = translate_quotes(tokens[i]) if types[i] == ?s }
-        token = tokens.join.strip.gsub(/\A=+/,'')
-      else
-        token = translate_quotes(quote(tokens.join.strip))
-      end
-    }
-    formulas.delete_if { |f| f == '""' }
+  def eval(string)
+    formulas = tokenize(string)
+    formulas.collect! { |f| f.collect { |t| t.value }.join }
     formulas.collect! { |f| Kernel.eval(f, binding) }
     formulas.size == 1 ? formulas.first : formulas
   end
@@ -261,8 +257,8 @@ class LXL::Parser
   #  quote('a "quoted" value.')
   #  # => '"a ""quoted"" value."'
   #
-  def quote(text)
-    '"'+text.to_s.gsub('"','""')+'"'
+  def quote(string)
+    '"'+string.to_s.gsub('"','""')+'"'
   end
   
   # Translate "" to \" in quoted strings.
@@ -270,58 +266,91 @@ class LXL::Parser
   #   translate_quotes('"a ""quoted"" value."')
   #   # => '"a \"quoted\" value."'
   #
-  def translate_quotes(text)
-    text.to_s.gsub(/([^\\])""/,'\1\"')
+  def translate_quotes(string)
+    string.to_s.gsub(/([^\\])""/,'\1\"')
   end
 
-  # Tokenize a formula into <tt>[TypesString, TokensArray]</tt>.
+  # Tokenize a string into an array of formulas (Array of Token).
   #--
   # const_missing is defined by Ruby, raises a NameError.
   # method_missing is not, Ruby uses user-defined if present, NoMethodError otherwise.
   #++
-  def tokenize(formula)
+  def tokenize(string)
+    formulas = [[]]
+    separators = [?=, ?;]
     ops = Hash[*EXCEL_OPERATORS.zip(RUBY_OPERATORS).flatten]
 
-    # Parse formula
-    types,tokens = @lexer.scan(formula.to_s)
+    # Translate formula prefixes to `= to avoid clashing with the = operator
+    # For lines that start with = (optional leading whitespace)
+    string.gsub!(/^\s*=/, '`=')
+    
+    # Parse string
+    types,tokens = @lexer.scan(string.to_s)
     raise SyntaxError, 'unbalanced parentheses' unless balanced?(types)
     
+    # Translate formula prefixes to `= to avoid clashing with the = operator
+    # For single line separations via semi-colon: semi-colon (optional whitespace) =
+    types.each_index do |i|
+      type,token = types[i], tokens[i]
+      if token == '='
+        last = types[0..i-1].reverse.find { |t| t != ?w }
+        types[i] = ?= if last == ?;
+      end
+    end
+
     # Parse tokens
-    tokens.each_index do |i|
-      type, token = types[i], tokens[i]
-      tokens[i] = case type
-        when ?o then ops[token]
-        when ?p then token.to_f/100
-        when ?f then token.to_f
-        when ?i then token.to_i
-        when ?r then
-          range = token.split(/:/)
-          range[1] = range[0] if range.size == 1
-          "self.class.range_class.new(*#{range.inspect})"
-        when ?t then
-          upper = token.to_s.upcase
-          lower = token.to_s.downcase
-          raise NoMethodError, "protected method `#{token}` called for #{self}" if @namespace.const_get(:METHODS).include?(lower)
-          custom_const_missing = begin [@namespace.const_get(:ConstMissing),true].last rescue false end
-          # Constants
-          if tokens[i+1] != '('
-            if @namespace.const_defined?(upper) or custom_const_missing
-              types[i] = ?C
-              "@namespace.const_get(:#{upper})"
-            else token; end
-          # Functions
-          else
-            if @namespace.respond_to?(lower) or @namespace.respond_to?(:method_missing)
-              types[i] = ?F
-              "@namespace.#{lower}"
-            else token; end
-          end
-        else token; end
+    last_type = ??
+    while tokens.size > 0
+      type,token = types.shift, tokens.shift
+      # Text
+      if formulas.last.size == 0 and last_type != ?= and type != ?=
+        text = []
+        while tokens.size > 0 and ! separators.include?(types.first)
+          types.shift
+          text << tokens.shift
+        end
+        text = text.join.chomp
+        formulas.last << Token.new(type, translate_quotes(quote(text))) unless text.empty?
+      # Formula
+      else
+        t = Token.new(type, token)
+        case type
+          when ?; then formulas << []
+          when ?= then formulas << []
+          when ?o then formulas.last << Token.new(type, ops[token])
+          when ?s then formulas.last << Token.new(type, translate_quotes(token))
+          when ?p then formulas.last << Token.new(type, token.to_f/100)
+          when ?f then formulas.last << Token.new(type, token.to_f)
+          when ?i then formulas.last << Token.new(type, token.to_i)
+          when ?r then
+            range = token.split(/:/)
+            range[1] = range[0] if range.size == 1
+            formulas.last << Token.new(type, "self.class.range_class.new(*#{range.inspect})")
+          when ?u then
+            upper = token.to_s.upcase
+            lower = token.to_s.downcase
+            raise NoMethodError, "protected method `#{token}` called for #{self}" if @namespace.const_get(:METHODS).include?(lower)
+            custom_const_missing = begin [@namespace.const_get(:ConstMissing),true].last rescue false end
+            # Constants
+            if tokens.first != '('
+              if @namespace.const_defined?(upper) or custom_const_missing
+                formulas.last << Token.new(?C, "@namespace.const_get(:#{upper})")
+              else formulas.last << t end
+            # Functions
+            else
+              if @namespace.respond_to?(lower) or @namespace.respond_to?(:method_missing)
+                formulas.last << Token.new(?F, "@namespace.#{lower}")
+              else formulas.last << t end
+            end
+          else formulas.last << t end
+      end
+      last_type = type
     end
 
-    [types,tokens]
+    formulas.reject! { |f| f.size == 0 }
+    formulas
   end
-  
+
   # Check that parentheses are balanced.
   #
   def balanced?(list)
@@ -352,7 +381,6 @@ end
 # 
 class LXL::Range < Range
 
-  # B3 | B3: | B3:D5 | Sheet1!B3:D5 | [Book1]Sheet1!B3:D5 | [file.xls]Sheet1!B3:D5
   EXCEL_RANGE = /\A(\[([\w\.]+)\])?((\w+)!)?([A-Z]+[1-9]+)(:([A-Z]+[1-9]+)?)?/i
   
   # Workbook name.
@@ -410,8 +438,8 @@ class LXL::Range < Range
 
   def each
     if excel?
-      Range.new(first_column, last_column).each do |column|
-        Range.new(first_cell, last_cell).each do |cell|
+      (first_column..last_column).each do |column|
+        (first_cell..last_cell).each do |cell|
           yield column+cell.to_s
         end
       end
@@ -498,7 +526,7 @@ end
 # 
 #   class MyLXLNamespace < LXL::Namespace
 #     NAME = 'John Doe'
-#     def upper(text) text.to_s.upcase end
+#     def upper(string) string.to_s.upcase end
 #   end
 #   
 #   class MyLXL < LXL::Parser
@@ -550,8 +578,8 @@ class LXL::Namespace < LXL::EmptyNamespace
     Date.new(y,m,d).ajd.to_f
   end
 
-  def datetime(text)
-    DateTime.parse(text.to_s).ajd.to_f
+  def datetime(string)
+    DateTime.parse(string.to_s).ajd.to_f
   end
 
   def time(h,m,s)
@@ -562,35 +590,24 @@ end
 
 # Based on John Carter's LittleLexer (http://littlelexer.rubyforge.org).
 #
-class LXL::Lexer #:nodoc: all
+class LXL::Lexer
 
   class LexerJammed < Exception; end
 
-  def initialize(re_to_chr, skip_whitespace=true)
+  def initialize(re_to_chr)
     @re_to_chr = re_to_chr
-    @skip_whitespace = skip_whitespace
-  end
-
-  def scan(string, string_tokens=nil)
-    types = String.new
-    tokens = Array.new
-    if string_tokens
-      next_token(string) do |t,token,tail|
-        types << t
-        tokens << [string_tokens[0...tail], string[0...tail]]
-        string = string[tail..-1]
-        string_tokens = string_tokens[tail..-1]
-      end
-    else
-      next_token(string) do |t,token,tail|
-        types << t
-        tokens << token
-      end
+  end
+
+  def scan(string)
+    types,tokens = [],[]
+    next_token(string) do |type,token|
+      types << type
+      tokens << token
     end
-    return types,tokens
+    [types,tokens]
   end
 
-  private
+  protected
 
   def next_token(string)
     match_data = nil
@@ -600,7 +617,7 @@ class LXL::Lexer #:nodoc: all
         match_data = re.match(string)
         next unless match_data
         token = match_data[0]
-        yield chr,token, match_data.end(0) unless chr == ?\s && @skip_whitespace 
+        yield chr, token
         string = match_data.post_match
         failed = false
         break
@@ -618,8 +635,8 @@ end
 if $0 == __FILE__
 
   formulas = %{
-    This is some text;
-    ="This is some ""quoted"" text"
+    This is a string;
+    ="This is a ""quoted"" string"
     =((1+2)*(10-6))/2
     =datetime("2004-11-22 11:11:00")=DATE(2004,11,22)+TIME(11,11,00)
     =IN(" is ", "this is a string")
@@ -638,6 +655,6 @@ if $0 == __FILE__
 
   # multiple formulas separated by semi-colon
   puts LXL.eval(formulas).inspect
-  # => ["This is some text", "This is some \"quoted\" text", 6, true, true, [1, "two", 3.0], true, false, true, "yes", "this and that", 8, 0.502]
+  # => ["This is a string", "This is a \"quoted\" string", 6, true, true, [1, "two", 3.0], true, false, true, "yes", "this and that", 8, 0.502]
 
 end

data/test/lxl_test.rb

@@ -5,7 +5,7 @@ require 'lxl'
 class MyNamespace < LXL::Namespace
   register_symbols :foo
   register_deferred :bar
-  def capitalize(text) text.to_s.capitalize end
+  def capitalize(string) string.to_s.capitalize end
 end
 
 class LXLTest < Test::Unit::TestCase
@@ -16,9 +16,9 @@ class LXLTest < Test::Unit::TestCase
 
   def test_multiple_formula
     formulas = %{
-      This is some text;
-      "This is some ""quoted"" text"
-      ="This is some ""quoted"" text"
+      This is a string;
+      "This is a ""quoted"" string"
+      ="This is a ""quoted"" string"
       =";"=";"
       =";"=":"
       =A
@@ -39,7 +39,7 @@ class LXLTest < Test::Unit::TestCase
       =0.2589=25.89%
       ="embedded percentages 25% and semi-colons ; are working properly"
     }
-    expected =  ["This is some text", "\"This is some \"quoted\" text\"", "This is some \"quoted\" text"]
+    expected =  ["This is a string", "\"This is a \"quoted\" string\"", "This is a \"quoted\" string"]
     expected += [true, false, :A, :B, false, 6, true, true, [1, 'two', 3.0], true, false, true, 'yes', 'this and that']
     expected += [8, 0.502, true, true, 'embedded percentages 25% and semi-colons ; are working properly']
     MyNamespace.register_symbols(:A, :B)
@@ -132,7 +132,7 @@ class LXLTest < Test::Unit::TestCase
 
   class MyLXLNamespace < LXL::Namespace
     NAME = 'John Doe'
-    def upper(text) text.to_s.upcase end
+    def upper(string) string.to_s.upcase end
   end
 
   class MyLXL < LXL::Parser

metadata

@@ -3,8 +3,8 @@ rubygems_version: 0.8.3
 specification_version: 1
 name: lxl
 version: !ruby/object:Gem::Version 
-  version: 0.3.8
-date: 2005-02-15
+  version: 0.4.0
+date: 2005-02-16
 summary: LXL (Like Excel) is a mini-language that mimics Microsoft Excel formulas. Easily extended with new constants and functions.
 require_paths: 
   - lib
@@ -36,8 +36,8 @@ files:
   - README.en
   - lib/lxl.rb
   - test/spreadsheet.rb
-  - test/lxl_test.rb
   - test/lxl_spreadsheet_test.rb
+  - test/lxl_test.rb
 test_files: 
   - test/lxl_test.rb
 rdoc_options: