excel_to_code 0.0.1

data/src/rewrite/rewrite_formulae_to_ast.rb

@@ -0,0 +1,24 @@
+require_relative '../excel/formula_peg'
+
+class RewriteFormulaeToAst
+  
+  def self.rewrite(input,output)
+    self.new.rewrite(input,output)
+  end
+  
+  # input should be in the form: 'thing\tthing\tformula\n' where the last field is always a forumla
+  # output will be in the form 'thing\tthing\tast\n'
+  def rewrite(input,output)
+    input.lines.with_index do |line,i|
+      line =~ /^(.*\t)(.*?)$/
+      output.write $1
+      ast =  Formula.parse($2)
+      if ast
+        output.puts ast.to_ast[1].to_s
+      else
+        $stderr.puts "Formula not parsed on line #{i+1}: #{line}"
+        output.puts "[:parse_error,#{line.inspect}]"
+      end
+    end
+  end
+end

data/src/rewrite/rewrite_merge_formulae_and_values.rb

@@ -0,0 +1,18 @@
+class RewriteMergeFormulaeAndValues
+  def self.rewrite(*args)
+    new.rewrite(*args)
+  end
+  
+  def rewrite(values,shared_formulae,array_formula,simple_formulae,output)
+    shared_formulae = Hash[shared_formulae.readlines.map { |line| [line[/(.*?)\t/,1],line]}]
+    array_formula = Hash[array_formula.readlines.map { |line| [line[/(.*?)\t/,1],line]}]
+    simple_formulae = Hash[simple_formulae.readlines.map { |line| [line[/(.*?)\t/,1],line]}]
+
+    values.lines do |line|
+      ref = line[/(.*?)\t/,1]
+      output.puts simple_formulae[ref] || array_formula[ref] || shared_formulae[ref] || line
+    end
+  end
+  
+end
+  

data/src/rewrite/rewrite_relationship_id_to_filename.rb

@@ -0,0 +1,22 @@
+class RewriteRelationshipIdToFilename
+  
+  def self.rewrite(*args)
+    self.new.rewrite(*args)
+  end
+  
+  def rewrite(input,relationships_file,output)
+    relationships_file.rewind
+    relationships_file.rewind
+    relationships = Hash[relationships_file.readlines.map { |line| line.split("\t")}]
+    input.lines do |line|
+      parts = line.split("\t")
+      rid = parts.pop.strip
+      if relationships.has_key?(rid)
+        output.puts "#{parts.join("\t")}#{parts.size > 0 ? "\t" : ""}#{relationships[rid].strip}"
+      else
+        $stderr.puts "Warning, #{rid.inspect} not found in relationships file #{relationships.inspect}"
+        outputs.puts "Warning, #{rid.inspect} not found in relationships file #{relationships.inspect}"
+      end
+    end
+  end
+end

data/src/rewrite/rewrite_shared_formulae.rb

@@ -0,0 +1,38 @@
+require_relative 'ast_copy_formula'
+
+class RewriteSharedFormulae
+  def self.rewrite(input,output)
+    new.rewrite(input,output)
+  end
+  
+  def rewrite(input,output)
+    input.lines do |line|
+      ref, copy_range, formula = line.split("\t")
+      share_formula(ref,formula,copy_range,output)
+    end
+  end
+  
+  def share_formula(ref,formula,copy_range,output)
+    shared_ast = eval(formula)
+    copier = AstCopyFormula.new
+    copy_range = Area.for(copy_range)
+    copy_range.calculate_excel_variables
+    
+    start_reference = copy_range.excel_start
+    
+    r = Reference.for(ref)
+    r.calculate_excel_variables
+    
+    offset_from_formula_to_start_rows = start_reference.excel_row_number - r.excel_row_number
+    offset_from_formula_to_start_columns = start_reference.excel_column_number - r.excel_column_number
+    
+    copy_range.offsets.each do |row,column|
+      new_ref = start_reference.offset(row,column)
+      copier.rows_to_move = row + offset_from_formula_to_start_rows
+      copier.columns_to_move = column + offset_from_formula_to_start_columns
+      ast = copier.copy(shared_ast)
+      output.puts "#{new_ref}\t#{ast.inspect}"
+    end
+  end
+  
+end

data/src/rewrite/rewrite_values_to_ast.rb

@@ -0,0 +1,28 @@
+require_relative '../excel/formula_peg'
+
+class RewriteValuesToAst
+  
+  def self.rewrite(input,output)
+    self.new.rewrite(input,output)
+  end
+  
+  # input should be in the form: 'thing\tthing\tformula\n' where the last field is always a forumla
+  # output will be in the form 'thing\tthing\tast\n'
+  def rewrite(input,output)
+    input.lines do |line|
+      line =~ /^(.*?)\t(.*?)\t(.*)\n/
+      ref, type, value = $1, $2, $3
+      ast = case type
+      when 'b'; value == "1" ? [:boolean_true] : [:boolean_false]
+      when 's'; [:shared_string,value]
+      when 'n'; [:number,value]
+      when 'e'; [:error,value]
+      when 'str'; [:string,value]
+      else
+        $stderr.puts "Type #{type} not known in #{line}"
+        [:parse_error,line.inspect]
+      end
+      output.puts "#{ref}\t#{ast}"
+    end
+  end
+end

data/src/rewrite/rewrite_whole_row_column_references_to_areas.rb

@@ -0,0 +1,90 @@
+require_relative '../excel'
+
+class WorksheetDimension
+  
+  attr_reader :first_column, :last_column, :first_row, :last_row
+  
+  def initialize(area)
+    @area = Area.for(area)
+    @area.calculate_excel_variables
+    @first_column = @area.excel_start.excel_column
+    @last_column =  @area.excel_finish.excel_column
+    @first_row = @area.excel_start.excel_row
+    @last_row = @area.excel_finish.excel_row
+  end
+  
+  def map_row(start,finish)
+    ["#{first_column}#{start}","#{last_column}#{finish}"]
+  end
+
+  def map_column(start,finish)
+    ["#{start}#{first_row}","#{finish}#{last_row}"]
+  end
+  
+end
+
+class MapColumnAndRowRangeAst 
+  
+  attr_accessor :default_worksheet_name, :worksheet_dimensions
+  
+  def initialize(default_worksheet_name,worksheet_dimensions)
+    @default_worksheet_name, @worksheet_dimensions = default_worksheet_name, worksheet_dimensions
+  end
+  
+  def map(ast)
+    if ast.is_a?(Array)
+      operator = ast.shift
+      if respond_to?(operator)
+        send(operator,*ast)
+      else
+        [operator,*ast.map {|a| map(a) }]
+      end
+    else
+      return ast
+    end
+  end
+  
+  def sheet_reference(sheet_name,reference)
+    if reference.first == :row_range
+      [:sheet_reference,sheet_name,[:area,*worksheet_dimensions[sheet_name].map_row(reference[1],reference[2])]]
+    elsif reference.first == :column_range
+      [:sheet_reference,sheet_name,[:area,*worksheet_dimensions[sheet_name].map_column(reference[1],reference[2])]]
+    else
+      [:sheet_reference,sheet_name,reference]
+    end
+  end
+  
+  def row_range(start,finish)
+    [:area,*worksheet_dimensions[default_worksheet_name].map_row(start,finish)]
+  end
+  
+  def column_range(start,finish)
+    [:area,*worksheet_dimensions[default_worksheet_name].map_column(start,finish)]
+  end
+  
+end
+
+class RewriteWholeRowColumnReferencesToAreas
+  
+  def self.rewrite(input,default_worksheet_name,worksheet_dimensions,output)
+    new.rewrite(input,default_worksheet_name,worksheet_dimensions,output)
+  end
+  
+  def rewrite(input,default_worksheet_name,worksheet_dimensions,output)
+    dimensions =  Hash[worksheet_dimensions.readlines.map do |line| 
+      worksheet_name, area = line.split("\t")
+      [worksheet_name,WorksheetDimension.new(area)]
+    end]
+    mapper = MapColumnAndRowRangeAst.new(default_worksheet_name,dimensions)
+    input.lines do |line|
+      if line =~ /(:column_range|:row_range)/
+        content = line.split("\t")
+        ast = eval(content.pop)
+        output.puts "#{content.join("\t")}\t#{mapper.map(ast).inspect}"
+      else
+        output.puts line
+      end
+    end
+  end
+  
+end

data/src/rewrite/rewrite_worksheet_names.rb

@@ -0,0 +1,20 @@
+class RewriteWorksheetNames
+  
+  def self.rewrite(worksheet_names,relationships,output)
+    self.new.rewrite(worksheet_names,relationships,output)
+  end
+  
+  # Expects worksheet names in the form:
+  # name\trelationship_id\n
+  # Expects relationships in the form:
+  # relationship_id\tfilename\n
+  # Outputs worksheet names in the form:
+  # name\tfilename\n
+  def rewrite(worksheet_names,relationships,output)
+    relationships = Hash[relationships.readlines.map { |line| line.split("\t")}]
+    worksheet_names.lines do |line|
+      rid, name = line.split("\t")
+      output.puts "#{name.strip}\t#{relationships[rid].strip}"
+    end
+  end
+end

data/src/simplify.rb

@@ -0,0 +1,16 @@
+require_relative "simplify/replace_shared_strings"
+require_relative "simplify/replace_named_references"
+require_relative "simplify/replace_table_references"
+require_relative "simplify/replace_ranges_with_array_literals"
+require_relative "simplify/replace_blanks"
+require_relative "simplify/inline_formulae"
+require_relative "simplify/replace_formulae_with_calculated_values"
+require_relative "simplify/replace_indirects_with_references"
+require_relative "simplify/simplify_arithmetic"
+require_relative "simplify/identify_dependencies"
+require_relative "simplify/remove_cells"
+require_relative "simplify/count_formula_references"
+require_relative "simplify/identify_repeated_formula_elements"
+require_relative "simplify/replace_common_elements_in_formulae"
+require_relative "simplify/replace_arrays_with_single_cells"
+require_relative "simplify/replace_values_with_constants"

data/src/simplify/count_formula_references.rb

@@ -0,0 +1,58 @@
+class CountFormulaReferences
+  
+  attr_accessor :references
+  attr_accessor :dependencies
+  attr_accessor :current_sheet
+  
+  def initialize(references = {}, dependencies = {})
+    @references = references
+    @dependencies = dependencies
+    dependencies.default_proc = lambda do |hash,key|
+      hash[key] = {}
+    end
+    @current_sheet = []
+  end
+  
+  def count(references)
+    @references = references
+    references.each do |sheet,cells|
+      current_sheet << sheet
+      cells.each do |ref,ast|
+        count_dependencies_for(sheet,ref,ast)
+      end
+      current_sheet.pop
+    end
+    return dependencies
+  end
+    
+  def count_dependencies_for(sheet,ref,ast)
+    current_sheet.push(sheet)
+    map(ast)
+    current_sheet.pop
+  end
+  
+  def map(ast)
+    return ast unless ast.is_a?(Array)
+    operator = ast[0]
+    if respond_to?(operator)
+      send(operator,*ast[1..-1])
+    else
+      ast[1..-1].each do |a|
+        map(a)
+      end
+    end
+  end
+  
+  def sheet_reference(sheet,reference)
+    ref = reference.last.gsub('$','')
+    @dependencies[sheet][ref] ||= 0
+    @dependencies[sheet][ref] += 1
+  end
+  
+  def cell(reference)
+    ref = reference.gsub('$','')
+    @dependencies[current_sheet.last][ref] ||= 0
+    @dependencies[current_sheet.last][ref] += 1
+  end
+   
+end

data/src/simplify/identify_dependencies.rb

@@ -0,0 +1,56 @@
+class IdentifyDependencies
+  
+  attr_accessor :references
+  attr_accessor :dependencies
+  attr_accessor :current_sheet
+  
+  def initialize(references = {}, dependencies = {})
+    @references = references
+    @dependencies = dependencies
+    dependencies.default_proc = lambda do |hash,key|
+      hash[key] = {}
+    end
+    @current_sheet = []
+  end
+  
+  def add_depedencies_for(sheet,cell = :all)
+    if cell == :all
+      return unless references.has_key?(sheet)
+      references[sheet].each do |ref,ast|
+        recursively_add_dependencies_for(sheet,ref)
+      end
+    else
+      recursively_add_dependencies_for(sheet,cell)
+    end
+  end
+    
+  def recursively_add_dependencies_for(sheet,cell)
+    return if dependencies[sheet].has_key?(cell)
+    dependencies[sheet][cell] = true
+    return unless references.has_key?(sheet)
+    ast = references[sheet][cell]
+    return unless ast
+    current_sheet.push(sheet)
+    map(ast)
+    current_sheet.pop
+  end
+  
+  def map(ast)
+    return ast unless ast.is_a?(Array)
+    operator = ast[0]
+    if respond_to?(operator)
+      send(operator,*ast[1..-1])
+    else
+      [operator,*ast[1..-1].map {|a| map(a) }]
+    end
+  end
+  
+  def sheet_reference(sheet,reference)
+    recursively_add_dependencies_for(sheet,reference.last.gsub('$',''))
+  end
+  
+  def cell(reference)
+    recursively_add_dependencies_for(current_sheet.last,reference.gsub('$',''))
+  end
+   
+end

data/src/simplify/identify_repeated_formula_elements.rb

@@ -0,0 +1,37 @@
+class IdentifyRepeatedFormulaElements
+  
+  attr_accessor :references
+  attr_accessor :counted_elements
+  attr_accessor :bothered_threshold
+  
+  def initialize
+    @references = {}
+    @counted_elements = {}
+    @counted_elements.default_proc = lambda do |hash,key|
+      hash[key] = 0
+    end
+    @bothered_threshold = 20
+  end
+  
+  def count(references)
+    @references = references
+    references.each do |sheet,cells|
+      cells.each do |ref,ast|
+        identify_repeated_formulae(ast)
+      end
+    end
+    return @counted_elements
+  end
+
+  
+  def identify_repeated_formulae(ast)
+    string = ast.to_s
+    return unless ast.is_a?(Array)
+    return if [:number,:string,:blank,:null,:error,:boolean_true,:boolean_false,:sheet_reference,:cell, :row].include?(ast.first)
+    return if string.length < bothered_threshold
+    @counted_elements[string] += 1
+    ast.each do |a|
+      identify_repeated_formulae(a)
+    end
+  end   
+end

data/src/simplify/inline_formulae.rb

@@ -0,0 +1,77 @@
+class InlineFormulaeAst
+  
+  attr_accessor :references, :current_sheet_name, :inline_ast
+  
+  def initialize(references, current_sheet_name, inline_ast = nil)
+    @references, @current_sheet_name, @inline_ast = references, [current_sheet_name], inline_ast
+    @inline_ast ||= lambda { |sheet,reference,references| true }
+  end
+  
+  def map(ast)
+    return ast unless ast.is_a?(Array)
+    operator = ast[0]
+    if respond_to?(operator)
+      send(operator,*ast[1..-1])
+    else
+      [operator,*ast[1..-1].map {|a| map(a) }]
+    end
+  end
+  
+  def sheet_reference(sheet,reference)
+    if inline_ast.call(sheet,reference.last.upcase.gsub('$',''),references)
+      ast = references[sheet][reference.last.upcase.gsub('$','')]
+      if ast
+        current_sheet_name.push(sheet)
+        result = map(ast)
+        current_sheet_name.pop
+      else
+        result = [:blank]
+      end
+    else
+      result = [:sheet_reference,sheet,reference]
+    end
+    result
+  end
+  
+  # TODO: Optimize by replacing contents of references hash with the inlined version
+  def cell(reference)
+    if inline_ast.call(current_sheet_name.last,reference.upcase.gsub('$',''),references)
+      ast = references[current_sheet_name.last][reference.upcase.gsub('$','')]
+      if ast
+        map(ast)
+      else
+        [:blank]
+      end
+    else
+      if current_sheet_name.size > 1
+        [:sheet_reference,current_sheet_name.last,[:cell,reference]]
+      else
+        [:cell,reference]
+      end
+    end
+  end
+    
+end
+  
+
+class InlineFormulae
+  
+  attr_accessor :references, :default_sheet_name, :inline_ast
+  
+  def self.replace(*args)
+    self.new.replace(*args)
+  end
+  
+  def replace(input,output)
+    rewriter = InlineFormulaeAst.new(references, default_sheet_name, inline_ast)
+    input.lines do |line|
+      # Looks to match lines with references
+      if line =~ /\[:cell/
+        ref, ast = line.split("\t")
+        output.puts "#{ref}\t#{rewriter.map(eval(ast)).inspect}"
+      else
+        output.puts line
+      end
+    end
+  end
+end