excel_to_code 0.1.23 → 0.2.0

data/src/simplify/identify_repeated_formula_elements.rb

@@ -15,21 +15,20 @@ class IdentifyRepeatedFormulaElements
   
   def count(references)
     @references = references
-    references.each do |sheet,cells|
-      cells.each do |ref,ast|
-        identify_repeated_formulae(ast)
-      end
+    references.each do |ref,ast|
+      identify_repeated_formulae(ast)
     end
     return @counted_elements
   end
 
+  IGNORE_TYPES = {:number => true, :string => true, :blank => true, :null => true, :error => true, :boolean_true => true, :boolean_false => true, :sheet_reference => true, :cell => true,  :row => true}
   
   def identify_repeated_formulae(ast)
     string = ast.to_s
     return unless ast.is_a?(Array)
-    return if [:number,:string,:blank,:null,:error,:boolean_true,:boolean_false,:sheet_reference,:cell, :row].include?(ast.first)
+    return if IGNORE_TYPES.has_key?(ast.first)
     return if string.length < bothered_threshold
-    @counted_elements[string] += 1
+    @counted_elements[ast] += 1
     ast.each do |a|
       identify_repeated_formulae(a)
     end

data/src/simplify/inline_formulae.rb

@@ -1,67 +1,67 @@
 class InlineFormulaeAst
+
+  BLANK = [:blank]
   
   attr_accessor :references, :current_sheet_name, :inline_ast
-  attr_accessor :replacements_made_in_the_last_pass
+  attr_accessor :count_replaced
   
-  def initialize(references, current_sheet_name, inline_ast = nil)
+  def initialize(references = nil, current_sheet_name = nil, inline_ast = nil)
     @references, @current_sheet_name, @inline_ast = references, [current_sheet_name], inline_ast
-    @replacements_made_in_the_last_pass = 0
-    @inline_ast ||= lambda { |sheet,reference,references| true }
+    @count_replaced = 0
+    @inline_ast ||= lambda { |sheet, ref, references| true } # Default is to always inline
   end
   
   def map(ast)
     return ast unless ast.is_a?(Array)
-    operator = ast[0]
-    if respond_to?(operator)
-      send(operator,*ast[1..-1])
-    else
-      [operator,*ast[1..-1].map {|a| map(a) }]
+    if respond_to?(ast[0])
+      send(ast[0], ast) 
+    else # In this case needs to be an else because don't want to map first argument in OFFSET(cell_to_offset_from_shouldn't_be_mapped, rows, columns)
+      ast.each { |a| map(a) }
     end
+    ast
   end
 
-  def function(name,*args)
-    case name
-    when 'OFFSET'
-      [:function, name, args.shift, *args.map { |a| map(a) }]
+  def function(ast)
+    if ast[1] == :OFFSET && (ast[2][0] == :cell || ast[2][0] == :sheet_reference)
+      # Don't map the second argument - it should be left as a cell refernce
+      ast[3..-1].each {|a| map(a) }
     else
-      [:function, name, *args.map { |a| map(a) }]
+      # Otherwise good to map all the other arguments
+      ast.each { |a| map(a) }
     end
   end
   
-  def sheet_reference(sheet,reference)
-    return [:error, '#REF!'] unless references[sheet]
-    if inline_ast.call(sheet,reference.last.upcase.gsub('$',''),references)
-      @replacements_made_in_the_last_pass += 1
-      ast = references[sheet][reference.last.upcase.gsub('$','')]
-      if ast
-        current_sheet_name.push(sheet)
-        result = map(ast)
-        current_sheet_name.pop
-      else
-        result = [:blank]
-      end
-    else
-      result = [:sheet_reference,sheet,reference]
-    end
-    result
+  # Should be of the form [:sheet_reference, sheet_name, reference]
+  # FIXME: Can we rely on reference always being a [:cell, ref] at this stage?
+  def sheet_reference(ast)
+    return unless ast[2][0] == :cell
+    sheet = ast[1].to_sym
+    ref = ast[2][1].to_s.upcase.gsub('$','').to_sym
+    # FIXME: Need to check if valid worksheet and return [:error, "#REF!"] if not
+    # Now check user preference on this
+    return unless inline_ast.call(sheet,ref, references)
+    @count_replaced += 1
+    ast_to_inline = references[[sheet, ref]]
+    return ast.replace(BLANK) unless ast_to_inline
+    current_sheet_name.push(sheet)
+    map(ast_to_inline)
+    current_sheet_name.pop
+    ast.replace(ast_to_inline)
   end
   
-  # TODO: Optimize by replacing contents of references hash with the inlined version
-  def cell(reference)
-    if inline_ast.call(current_sheet_name.last,reference.upcase.gsub('$',''),references)
-      @replacements_made_in_the_last_pass += 1
-      ast = references[current_sheet_name.last][reference.upcase.gsub('$','')]
-      if ast
-        map(ast)
-      else
-        [:blank]
-      end
-    else
-      if current_sheet_name.size > 1
-        [:sheet_reference,current_sheet_name.last,[:cell,reference]]
-      else
-        [:cell,reference]
-      end
+  # Format [:cell, ref]
+  def cell(ast)
+    sheet = current_sheet_name.last
+    ref = ast[1].to_s.upcase.gsub('$', '').to_sym
+    if inline_ast.call(sheet, ref, references)
+      @count_replaced += 1
+      ast_to_inline = references[[sheet, ref]]
+      return ast.replace(BLANK) unless ast_to_inline
+      map(ast_to_inline)
+      ast.replace(ast_to_inline)
+    # FIXME: Check - is this right? does it work recursively enough?
+    elsif current_sheet_name.size > 1 
+      ast.replace([:sheet_reference, sheet, ast.dup])
     end
   end
     
@@ -76,7 +76,7 @@ class InlineFormulae
     self.new.replace(*args)
   end
 
-  attr_accessor :replacements_made_in_the_last_pass
+  attr_accessor :count_replaced
   
   def replace(input,output)
     rewriter = InlineFormulaeAst.new(references, default_sheet_name, inline_ast)
@@ -89,6 +89,6 @@ class InlineFormulae
         output.puts line
       end
     end
-    @replacements_made_in_the_last_pass = rewriter.replacements_made_in_the_last_pass
+    @count_replaced = rewriter.count_replaced
   end
 end

data/src/simplify/map_formulae_to_values.rb

@@ -16,115 +16,214 @@ class MapFormulaeToValues
     @value_for_ast = MapValuesToRuby.new
     @calculator = FormulaeCalculator.new
     @replacements_made_in_the_last_pass = 0
-    @cache = {}
   end
 
-  def reset
-    @cache = {}
+  def original_excel_filename=(new_filename)
+    @original_excel_filename = new_filename
+    @calculator.original_excel_filename = new_filename
   end
 
-  # FIXME: Caching works in the odd edge cases of long formula
-  # but I really need to find the root cause of the problem
-  def map(ast)
-    @calculator.original_excel_filename = original_excel_filename
-    @cache[ast] ||= do_map(ast)
+  def reset
+    # Not used any more
+    # FIXME: Remove references to this method
   end
 
-  def do_map(ast)
-    return ast unless ast.is_a?(Array)
-    operator = ast[0]
-    if respond_to?(operator)
-      send(operator,*ast[1..-1])
-    else
-      [operator,*ast[1..-1].map {|a| map(a) }]
-    end
+  DO_NOT_MAP = {:number => true, :string => true, :blank => true, :null => true, :error => true, :boolean_true => true, :boolean_false => true, :sheet_reference => true, :cell => true}
+
+  def map(ast)
+    ast[1..-1].each do |a| 
+      next unless a.is_a?(Array)
+      next if DO_NOT_MAP[(a[0])]
+      map(a)
+    end # Depth first best in this case?
+    send(ast[0], ast) if respond_to?(ast[0])
+    ast
   end
 
-  def prefix(operator,argument)
-    argument_value = value(map(argument))
-    return [:prefix, operator, map(argument)] if argument_value == :not_a_value
-    return ast_for_value(argument_value || 0) if operator == "+"
-    ast_for_value(@calculator.negative(argument_value))
+  # [:prefix, operator, argument]
+  def prefix(ast)
+    operator, argument = ast[1], ast[2]
+    argument_value = value(argument)
+    return if argument_value == :not_a_value
+    return ast.replace(ast_for_value(argument_value || 0)) if operator == "+"
+    ast.replace(ast_for_value(@calculator.negative(argument_value)))
   end
   
-  def arithmetic(left,operator,right)
-    l = value(map(left))
-    r = value(map(right))
-    if (l != :not_a_value) && (r != :not_a_value)
-      formula_value(operator.last,l,r)
-    else
-      [:arithmetic,map(left),operator,map(right)]
+  # [:arithmetic, left, operator, right]
+  def arithmetic(ast)
+    left, operator, right = ast[1], ast[2], ast[3]
+    l = @calculator.number_argument(value(left))
+    r = @calculator.number_argument(value(right))
+    if (l == :not_a_value) && (r == :not_a_value)
+      return ast
+    elsif (l != :not_a_value) && (r != :not_a_value)
+      ast.replace(formula_value(operator.last,l,r))
+    # SPECIAL CASES
+    elsif l == 0
+      case operator.last
+      when :+ 
+        ast.replace(right)
+      when :*, :/, :^
+        ast.replace([:number, 0])
+      end
+    elsif r == 0
+      case operator.last
+      when :+, :-
+        ast.replace(left)
+      when :* 
+        ast.replace([:number, 0])
+      when :/
+        ast.replace([:error, :'#DIV/0!'])
+      when :^
+        ast.replace([:number, 1])
+      end
+    elsif l == 1
+      case operator.last
+      when :*
+        ast.replace(right)
+      when :^
+        ast.replace([:number, 1])
+      end
+    elsif r == 1
+      case operator.last
+      when :*, :/, :^
+        ast.replace(left)
+      end
     end
+    ast
   end
-  
-  alias :comparison :arithmetic
 
-  def percentage(number)
-    ast_for_value(value([:percentage, number]))
+  def comparison(ast)
+    left, operator, right = ast[1], ast[2], ast[3]
+    l = value(left)
+    r = value(right)
+    return ast if (l == :not_a_value) || (r == :not_a_value)
+    ast.replace(formula_value(operator.last,l,r))
   end
-  
-  def string_join(*args)
-    values = args.map { |a| value(map(a)) } # FIXME: These eval statements are really bugging me. Must find a better solution
-    if values.any? { |a| a == :not_a_value }
-      [:string_join,*args.map { |a| map(a) }]
-    else
-      ast_for_value(@calculator.string_join(*values))
-    end
+
+  # [:percentage, number]
+  def percentage(ast)
+    ast.replace(ast_for_value(value([:percentage, ast[1]])))
   end
   
-  FUNCTIONS_THAT_SHOULD_NOT_BE_CONVERTED = %w{TODAY RAND RANDBETWEEN INDIRECT}
+  # [:string_join, stringA, stringB, ...]
+  def string_join(ast)
+    values = ast[1..-1].map { |a| value(a) } 
+    return if values.any? { |a| a == :not_a_value }
+    ast.replace(ast_for_value(@calculator.string_join(*values)))
+  end
   
-  def function(name,*args)
-    if FUNCTIONS_THAT_SHOULD_NOT_BE_CONVERTED.include?(name)
-      [:function,name,*args.map { |a| map(a) }]
-    elsif respond_to?("map_#{name.downcase}")
-      send("map_#{name.downcase}",*args)
+  # [:function, function_name, arg1, arg2, ...]
+  def function(ast)
+    name = ast[1]
+    return if name == :INDIRECT
+    return if name == :OFFSET
+    return if name == :COLUMN
+    if respond_to?("map_#{name.to_s.downcase}")
+      send("map_#{name.to_s.downcase}",ast)
     else
-      values = args.map { |a| value(map(a)) }
-      if values.any? { |a| a == :not_a_value }
-        [:function,name,*args.map { |a| map(a) }]
-      else
-        formula_value(name,*values)
-      end
+      values = ast[2..-1].map { |a| value(a) }
+      return if values.any? { |a| a == :not_a_value }
+      ast.replace(formula_value(name,*values))
     end
   end
 
-  def map_count(range)
-    return [:function, "COUNT", range] unless [:array, :cell, :sheet_reference].include?(range.first)
+  # [:function, "COUNT", range]
+  def map_count(ast)
+    range = ast[2]
+    return unless [:array, :cell, :sheet_reference].include?(range.first)
     range = array_as_values(range)
-    ast_for_value(range.size * range.first.size)
+    ast.replace(ast_for_value(range.size * range.first.size))
   end
   
-  def map_index(array,row_number,column_number = :not_specified)
-    return map_index_with_only_two_arguments(array,row_number) if column_number == :not_specified
+  # [:function, "INDEX", array, row_number, column_number]
+  def map_index(ast)
+    return map_index_with_only_two_arguments(ast) if ast.length == 4
 
-    array_mapped = map(array)
-    row_as_number = value(map(row_number))
-    column_as_number = value(map(column_number))
+    array_mapped = ast[2] 
+    row_as_number = value(ast[3])
+    column_as_number = value(ast[4])
 
-    return [:function, "INDEX", array_mapped, map(row_number), map(column_number)] if row_as_number == :not_a_value || column_as_number == :not_a_value
+    return if row_as_number == :not_a_value 
+    return if column_as_number == :not_a_value
 
-    array_as_values = array_as_values(array)
-    return  [:function, "INDEX", array_mapped, map(row_number), map(column_number)] unless array_as_values
+    array_as_values = array_as_values(array_mapped)
+    return unless array_as_values
 
-    result = @calculator.send(MapFormulaeToRuby::FUNCTIONS["INDEX"],array_as_values,row_as_number,column_as_number)
+    result = @calculator.send(MapFormulaeToRuby::FUNCTIONS[:INDEX],array_as_values,row_as_number,column_as_number)
     result = [:number, 0] if result == [:blank]
     result = ast_for_value(result)
-    result    
+    ast.replace(result)
   end
   
-  def map_index_with_only_two_arguments(array,row_number)
-    array_mapped = map(array)
-    row_as_number = value(map(row_number))
-    return [:function, "INDEX", array_mapped, map(row_number)] if row_as_number == :not_a_value
-    array_as_values = array_as_values(array)
-    return  [:function, "INDEX", array_mapped, map(row_number)] unless array_as_values
-    result = @calculator.send(MapFormulaeToRuby::FUNCTIONS["INDEX"],array_as_values,row_as_number)
+  # [:function, "INDEX", array, row_number]
+  def map_index_with_only_two_arguments(ast)
+    array_mapped = ast[2]
+    row_as_number = value(ast[3])
+    return if row_as_number == :not_a_value
+    array_as_values = array_as_values(array_mapped)
+    return unless array_as_values
+    result = @calculator.send(MapFormulaeToRuby::FUNCTIONS[:INDEX],array_as_values,row_as_number)
     result = [:number, 0] if result == [:blank]
     result = ast_for_value(result)
-    result
+    ast.replace(result)
   end
-  
+
+  # [:function, :SUM, a, b, c...] 
+  def map_sum(ast)
+      values = ast[2..-1].map { |a| value(a) }
+      return partially_map_sum(ast) if values.any? { |a| a == :not_a_value }
+      ast.replace(formula_value(:SUM,*values))
+  end
+
+  def partially_map_sum(ast)
+    number_total = 0
+    not_number_array = []
+    ast[2..-1].each do |a|
+      result = filter_numbers_and_not(a)
+      number_total += result.first
+      not_number_array.concat(result.last)
+    end
+    if number_total == 0 && not_number_array.empty?
+      ast.replace([:number, number_total])
+    # FIXME: Will I be haunted by this? What if doing a sum of something that isn't a number
+    # and so what is expected is a VALUE error?. YES. This doesn't work well.
+    #elsif number_total == 0 && not_number_array.size == 1
+    #  p not_number_array[0]
+    #  ast.replace(not_number_array[0])
+    else
+      new_ast = [:function, :SUM].concat(not_number_array)
+      new_ast.push([:number, number_total]) unless number_total == 0
+      ast.replace(new_ast)
+    end
+    ast
+  end
+
+  def filter_numbers_and_not(ast)
+    number_total = 0
+    not_number_array = []
+    case ast.first
+    when :array
+      array_as_values(ast).each do |row|
+        row.each do |c|
+          result = filter_numbers_and_not(c)
+          number_total += result.first
+          not_number_array.concat(result.last)
+        end
+      end
+    when :blank, :number, :percentage, :string, :boolean_true, :boolean_false
+      number = @calculator.number_argument(value(ast))
+      if number.is_a?(Symbol)
+        not_number_array.push(ast)
+      else
+        number_total += number
+      end
+    else
+      not_number_array.push(ast)
+    end
+    [number_total, not_number_array]
+  end
+
   def array_as_values(array_mapped)
     case array_mapped.first
     when :array
@@ -139,11 +238,29 @@ class MapFormulaeToValues
       nil
     end
   end
+
+  ERRORS = {
+    :"#NAME?" => :name,
+    :"#VALUE!" => :value,
+    :"#DIV/0!" => :div0,
+    :"#REF!" => :ref,
+    :"#N/A" => :na,
+    :"#NUM!" => :num
+  }
     
   def value(ast)
     return extract_values_from_array(ast) if ast.first == :array
-    return :not_a_value unless @value_for_ast.respond_to?(ast.first)
-    eval(@value_for_ast.send(*ast))
+    case ast.first
+    when :blank; nil
+    when :null; nil
+    when :number; ast[1]
+    when :percentage; ast[1]/100.0
+    when :string; ast[1]
+    when :error; ERRORS[ast[1]]
+    when :boolean_true; true
+    when :boolean_false; false
+    else return :not_a_value
+    end
   end
   
   def extract_values_from_array(ast)
@@ -157,15 +274,15 @@ class MapFormulaeToValues
   end
   
   def formula_value(ast_name,*arguments)
-    raise NotSupportedException.new("#{ast_name.inspect} function not recognised in #{MapFormulaeToRuby::FUNCTIONS.inspect}") unless MapFormulaeToRuby::FUNCTIONS.has_key?(ast_name)
+    raise NotSupportedException.new("#{ast_name} function not recognised in #{MapFormulaeToRuby::FUNCTIONS.inspect}") unless MapFormulaeToRuby::FUNCTIONS.has_key?(ast_name)
     ast_for_value(@calculator.send(MapFormulaeToRuby::FUNCTIONS[ast_name],*arguments))
   end
   
   def ast_for_value(value)
     return value if value.is_a?(Array) && value.first.is_a?(Symbol)
     @replacements_made_in_the_last_pass += 1
-    case value
-    when Numeric; [:number,value.inspect]
+    ast = case value
+    when Numeric; [:number,value]
     when true; [:boolean_true]
     when false; [:boolean_false]
     when Symbol; 
@@ -177,6 +294,7 @@ class MapFormulaeToValues
     else
       raise NotSupportedException.new("Ast for #{value.inspect} of class #{value.class} not recognised")
     end
+    CachingFormulaParser.map(ast)
   end
   
 end