excel_to_code 0.0.1

data/README

@@ -0,0 +1,41 @@
+# excel_to_code
+
+Converts some excel spreadsheets (.xlsx, not .xls) into some other programming languages (currently ruby or c).
+This allows the excel spreadsheets to be run programatically, without excel.
+
+Its cannonical source is at http://github.com/tamc/excel2code
+
+# Running excel_to_code
+
+To just have a go:
+
+	./bin/excel_to_c <excel_file_name>
+	
+NB:For small spreadsheets this will take a minute or so. For large spreadsheets it is best to run it overnight.
+	
+for more detail:
+	
+	./bin/excel_to_c --compile --run-tests --settable <name of input worksheet> --prune-except <name of output worksheet> <excel file name> 
+	
+this should work:
+
+	./bin/excel_to_c --help
+
+# Testing excel_to_code
+
+1. Make sure you have ruby 1.9.2 or later installed
+2. gem install bundler # May need to use sudo
+3. bundle
+4. rspec spec/*
+
+# Hacking excel_to_code
+
+There are some how to guides in the doc folder. 
+
+# Limitations
+
+1. Not tested at all on Windows
+2. INDIRECT formula must be convertable at runtime into a standard formula
+3. Doesn't implement all functions (see doc/Which_functions_are_implemented.md)
+4. Doesn't implement references that involve range unions and lists
+5. Sometimes gives cells as being empty, when excel would give the cell as having a numeric value of zero

data/bin/excel_to_c

@@ -0,0 +1,63 @@
+#!/usr/bin/env ruby
+require 'optparse'
+require_relative '../src/commands/excel_to_c'
+
+command = ExcelToC.new
+
+opts = OptionParser.new do |opts|
+  opts.banner = <<-END
+
+A command to approximately translate excel files into c code.
+Usage: excel_to_c [options] input_excel_file <output_directory>
+  input_excel_file is the name of a .xlsx excel file (i.e., produced by Excel 2007+, not an xls file) 
+  output_directory is the name of a folder in which to place the generated code. If not specified will have the same name as the input_excel_file, without the .xlsx
+  
+Support: http://github.com/tamc/excel2code
+END
+  
+  opts.separator ""
+  opts.separator "Specific options:"
+  
+  opts.on('-s','--settable WORKSHEET','Make it possible to set the values of cells in this worksheet at runtime. By default no values are settable.') do |sheet|
+    command.cells_that_can_be_set_at_runtime = { sheet => :all }
+  end
+
+  opts.on('-o','--output-name NAME','Filename to give to c version of code (and associated ruby interface). Defaults to a folder with the same name as the excel file.') do |name|
+    command.output_name = name
+  end
+  
+  opts.on('-p','--prune-except WORKSHEET',"Remove all cells except those on this worksheet, or that are required to calculate values on that worksheet. By default keeps all cells.") do |sheet|
+    command.cells_to_keep = { sheet => :all }
+  end
+  
+  opts.on('-c','--compile',"Compile the generated c code") do
+    command.actually_compile_c_code = true
+  end
+  
+  opts.on('-r','--run-tests',"Compile the generated c code and then run the tests") do
+    command.actually_compile_c_code = true
+    command.actually_run_tests = true
+  end
+  
+  opts.on("-h", "--help", "Show this message") do
+    puts opts
+    exit
+  end
+end
+
+begin
+  opts.parse!(ARGV)
+rescue OptionParser::ParseError => e 
+  STDERR.puts e.message, "\n", opts 
+  exit(-1)
+end
+
+unless ARGV.size > 0
+  puts opts
+  exit(-1)
+end
+
+command.excel_file = ARGV[0]
+command.output_directory = ARGV[1] if ARGV[1]
+
+command.go!

data/bin/excel_to_ruby

@@ -0,0 +1,9 @@
+#!/usr/bin/env ruby
+require_relative '../src/commands/excel_to_ruby'
+command = ExcelToRuby.new
+command.excel_file = ARGV[0]
+command.compiled_module_name = "ExcelSpreadsheet"
+command.output_directory = ARGV[1]
+command.values_that_can_be_set_at_runtime = { ARGV[2] => :all } if ARGV[2]
+command.outputs_to_keep = { ARGV[3] => :all } if ARGV[3]
+command.go!

data/src/commands.rb

@@ -0,0 +1,2 @@
+require_relative "commands/excel_to_ruby"
+require_relative "commands/excel_to_c"

data/src/commands/excel_to_c.rb

@@ -0,0 +1,858 @@
+# coding: utf-8
+
+require 'fileutils'
+require_relative '../util'
+require_relative '../excel'
+require_relative '../extract'
+require_relative '../rewrite'
+require_relative '../simplify'
+require_relative '../compile'
+
+# Used to throw normally fatal errors
+class ExcelToCException < Exception; end
+
+class ExcelToC
+  
+  # Required attribute. The source excel file. This must be .xlsx not .xls
+  attr_accessor :excel_file
+  
+  # Optional attribute. The output directory.
+  #  If not specified, will be '#{excel_file_name}/c'
+  attr_accessor :output_directory
+  
+  # Optional attribute. The name of the resulting c file (and associated ruby ffi module). Defaults to excelspreadsheet
+  attr_accessor :output_name
+
+  # Optional attribute. The excel file will be translated to xml and stored here.
+  # If not specified, will be '#{excel_file_name}/xml'
+  attr_accessor :xml_directory
+
+  # Optional attribute. The intermediate workings will be stored here.
+  # If not specified, will be '#{excel_file_name}/intermediate'
+  attr_accessor :intermediate_directory
+  
+  # Optional attribute. Specifies which cells have setters created in the c code so their values can be altered at runtime.
+  # It is a hash. The keys are the sheet names. The values are either the symbol :all to specify that all cells on that sheet 
+  # should be setable, or an array of cell names on that sheet that should be settable (e.g., A1)
+  attr_accessor :cells_that_can_be_set_at_runtime
+  
+  # Optional attribute. Specifies which cells must appear in the final generated code.
+  # The default is that all cells in the original spreadsheet appear in the final code.
+  #
+  # If specified, then any cells that are not:
+  #    * specified
+  #    * required to calculate one of the cells that is specified
+  #    * specified as a cell that can be set at runtime
+  # may be excluded from the final generated code.
+  #
+  # It is a hash. The keys are the sheet names. The values are either the symbol :all to specify that all cells on that sheet 
+  # should be lept, or an array of cell names on that sheet that should be kept (e.g., A1)
+  attr_accessor :cells_to_keep
+  
+  # Optional attribute. Boolean. 
+  #   * true - the generated c code is compiled
+  #   * false - the generated c code is not compiled (default, unless actuall_run_tests is specified as true)
+  attr_accessor :actually_compile_c_code
+
+  # Optional attribute. Boolean. 
+  #   * true - the generated tests are run
+  #   * false - the generated tests are not run
+  attr_accessor :actually_run_tests
+  
+  def set_defaults
+    raise ExcelToCException.new("No excel file has been specified in ExcelToC.excel_file") unless excel_file
+    
+    self.output_directory ||= File.join(File.dirname(excel_file),File.basename(excel_file,".*"),'c')
+    self.xml_directory ||= File.join(File.dirname(excel_file),File.basename(excel_file,".*"),'xml')
+    self.intermediate_directory ||= File.join(File.dirname(excel_file),File.basename(excel_file,".*"),'intermediate')
+    
+    self.output_name ||= "Excelspreadsheet"
+    
+    self.cells_that_can_be_set_at_runtime ||= {}
+    
+    # Make sure that all the cell names are downcase and don't have any $ in them
+    cells_that_can_be_set_at_runtime.keys.each do |sheet|
+      next unless cells_that_can_be_set_at_runtime[sheet].is_a?(Array)
+      cells_that_can_be_set_at_runtime[sheet] = cells_that_can_be_set_at_runtime[sheet].map { |reference| reference.gsub('$','').downcase }
+    end
+
+    if cells_to_keep
+      cells_to_keep.keys.each do |sheet|
+        next unless cells_to_keep[sheet].is_a?(Array)
+        cells_to_keep[sheet] = cells_to_keep[sheet].map { |reference| reference.gsub('$','').downcase }
+      end
+    end  
+    
+    self.excel_file = File.expand_path(excel_file)
+    self.output_directory = File.expand_path(output_directory)
+      
+  end
+  
+  def go!
+    # This sorts out the attributes
+    set_defaults
+    
+    # These turn the excel into a more accesible format
+    sort_out_output_directories
+    unzip_excel
+    
+    # These get all the information out of the excel and put
+    # into a useful format
+    extract_data_from_workbook
+    extract_data_from_worksheets
+    merge_table_files
+    rewrite_worksheets
+    
+    # These perform a series of transformations to the information
+    # with the intent of removing any redundant calculations
+    # that are in the excel
+    simplify_worksheets
+    optimise_and_replace_indirect_loop
+    replace_blanks
+    remove_any_cells_not_needed_for_outputs
+    inline_formulae_that_are_only_used_once
+    separate_formulae_elements
+    
+    # These actually create the code version of the excel
+    write_out_excel_as_code
+    write_build_script
+    write_fuby_ffi_interface
+    write_tests
+    
+    # These compile and run the code version of the excel
+    compile_c_code
+    run_tests
+    
+    puts
+    puts "The generated code is available in #{File.join(output_directory)}"
+  end
+  
+  def sort_out_output_directories    
+    FileUtils.mkdir_p(output_directory)
+    FileUtils.mkdir_p(xml_directory)
+    FileUtils.mkdir_p(intermediate_directory)
+  end
+  
+  def unzip_excel
+    puts `rm -fr '#{xml_directory}'`
+    puts `unzip -uo '#{excel_file}' -d '#{xml_directory}'`
+  end
+
+  def extract_data_from_workbook
+    extract_shared_strings
+    
+    extract ExtractNamedReferences, 'workbook.xml', 'named_references'
+    rewrite RewriteFormulaeToAst, 'named_references', 'named_references.ast'
+    
+    extract ExtractWorksheetNames, 'workbook.xml', 'worksheet_names_without_filenames'
+    extract ExtractRelationships, File.join('_rels','workbook.xml.rels'), 'workbook_relationships'
+    rewrite RewriteWorksheetNames, 'worksheet_names_without_filenames', 'workbook_relationships', 'worksheet_names'
+    rewrite MapSheetNamesToCNames, 'worksheet_names', 'worksheet_c_names'
+    
+    extract_dimensions_from_worksheets
+  end
+  
+  def extract_shared_strings
+    if File.exists?(File.join(xml_directory,'xl','sharedStrings.xml'))
+      extract ExtractSharedStrings, 'sharedStrings.xml', 'shared_strings'
+    else
+      FileUtils.touch(File.join(intermediate_directory,'shared_strings'))
+    end
+  end
+  
+  def extract_dimensions_from_worksheets    
+    dimension_file = intermediate('dimensions')
+    worksheets("Extracting dimensions") do |name,xml_filename|
+      dimension_file.write name
+      dimension_file.write "\t"
+      extract ExtractWorksheetDimensions, File.open(xml_filename,'r'), dimension_file 
+    end
+    dimension_file.close
+  end
+  
+  def extract_data_from_worksheets
+    worksheets("Initial data extract") do |name,xml_filename|
+      worksheet_directory = File.join(intermediate_directory,name)
+      FileUtils.mkdir_p(worksheet_directory)
+      worksheet_xml = File.open(xml_filename,'r')
+      { ExtractValues => 'values', 
+        ExtractSimpleFormulae => 'simple_formulae',
+        ExtractSharedFormulae => 'shared_formulae',
+        ExtractArrayFormulae => 'array_formulae'
+      }.each do |_klass,output_filename|
+        worksheet_xml.rewind
+        extract _klass, worksheet_xml, File.join(name,output_filename)
+        if _klass == ExtractValues
+          rewrite RewriteValuesToAst, File.join(name,output_filename), File.join(name,"#{output_filename}.ast")
+        else
+          rewrite RewriteFormulaeToAst, File.join(name,output_filename), File.join(name,"#{output_filename}.ast")
+        end  
+      end
+      worksheet_xml.rewind
+      extract ExtractWorksheetTableRelationships, worksheet_xml, File.join(name,'table_rids')
+      if File.exists?(File.join(xml_directory,'xl','worksheets','_rels',"#{File.basename(xml_filename)}.rels"))
+        extract ExtractRelationships, File.join('worksheets','_rels',"#{File.basename(xml_filename)}.rels"), File.join(name,'relationships')
+        rewrite RewriteRelationshipIdToFilename, File.join(name,'table_rids'), File.join(name,'relationships'), File.join(name,'table_filenames')
+        tables = intermediate(name,'tables')
+        table_extractor = ExtractTable.new(name)
+        table_filenames = input(name,'table_filenames')
+        table_filenames.lines.each do |line|
+          extract table_extractor, File.join('worksheets',line.strip), tables
+        end
+        close(tables,table_filenames)
+      else
+        FileUtils.touch File.join(intermediate_directory,name,'relationships')
+        FileUtils.touch File.join(intermediate_directory,name,'table_filenames')      
+        FileUtils.touch File.join(intermediate_directory,name,'tables')      
+      end
+      close(worksheet_xml)
+    end
+  end
+  
+  def rewrite_worksheets
+    worksheets("Initial rewrite of references and formulae") do |name,xml_filename|
+        rewrite_row_and_column_references(name,xml_filename)
+        rewrite_shared_formulae(name,xml_filename)
+        rewrite_array_formulae(name,xml_filename)
+        combine_formulae_files(name,xml_filename)
+    end
+  end
+  
+  def rewrite_row_and_column_references(name,xml_filename)
+    dimensions = input('dimensions')
+    %w{simple_formulae.ast shared_formulae.ast array_formulae.ast}.each do |file|
+      dimensions.rewind
+      i = File.open(File.join(intermediate_directory,name,file),'r')
+      o = File.open(File.join(intermediate_directory,name,"#{file}-nocols"),'w')
+      RewriteWholeRowColumnReferencesToAreas.rewrite(i,name, dimensions, o)
+      close(i,o)
+    end
+    dimensions.close
+  end
+  
+  def rewrite_shared_formulae(name,xml_filename)
+    i = File.open(File.join(intermediate_directory,name,'shared_formulae.ast-nocols'),'r')
+    o = File.open(File.join(intermediate_directory,name,"shared_formulae-expanded.ast"),'w')
+    RewriteSharedFormulae.rewrite(i,o)
+    close(i,o)
+  end
+  
+  def rewrite_array_formulae(name,xml_filename)
+    r = ReplaceNamedReferences.new
+    r.sheet_name = name
+    replace r, File.join(name,'array_formulae.ast-nocols'), 'named_references.ast', File.join(name,"array_formulae1.ast")
+
+    r = ReplaceTableReferences.new
+    r.sheet_name = name    
+    replace r, File.join(name,'array_formulae1.ast'), 'all_tables', File.join(name,"array_formulae2.ast")
+    replace SimplifyArithmetic, File.join(name,'array_formulae2.ast'), File.join(name,'array_formulae3.ast')
+    replace ReplaceRangesWithArrayLiterals, File.join(name,"array_formulae3.ast"), File.join(name,"array_formulae4.ast")
+    rewrite RewriteArrayFormulaeToArrays, File.join(name,"array_formulae4.ast"), File.join(name,"array_formulae5.ast")
+    rewrite RewriteArrayFormulae, File.join(name,'array_formulae5.ast'), File.join(name,"array_formulae-expanded.ast")
+  end
+  
+  def combine_formulae_files(name,xml_filename)
+    values = File.join(name,'values.ast')
+    shared_formulae = File.join(name,"shared_formulae-expanded.ast")
+    array_formulae = File.join(name,"array_formulae-expanded.ast")
+    simple_formulae = File.join(name,"simple_formulae.ast-nocols")
+    output = File.join(name,'formulae.ast')
+    rewrite RewriteMergeFormulaeAndValues, values, shared_formulae, array_formulae, simple_formulae, output
+  end
+  
+  def merge_table_files
+    tables = []
+    worksheets("Merging table files") do |name,xml_filename|
+      tables << File.join(intermediate_directory,name,'tables')
+    end
+    `sort #{tables.map { |t| " '#{t}' "}.join} > #{File.join(intermediate_directory,'all_tables')}`
+  end
+  
+  def simplify_worksheets
+    worksheets("Simplifying") do |name,xml_filename|
+      simplify_worksheet(name,xml_filename)
+    end
+  end
+  
+  def simplify_worksheet(name,xml_filename)
+    replace SimplifyArithmetic, File.join(name,'formulae.ast'), File.join(name,'formulae_simple_arithmetic.ast')
+    replace ReplaceSharedStrings, File.join(name,'formulae_simple_arithmetic.ast'), 'shared_strings', File.join(name,"formulae_no_shared_strings.ast")
+    replace ReplaceSharedStrings, File.join(name,'values.ast'), 'shared_strings', File.join(name,"values_no_shared_strings.ast")
+    r = ReplaceNamedReferences.new
+    r.sheet_name = name
+    replace r, File.join(name,'formulae_no_shared_strings.ast'), 'named_references.ast', File.join(name,"formulae_no_named_references.ast")
+
+    r = ReplaceTableReferences.new
+    r.sheet_name = name    
+    replace r, File.join(name,'formulae_no_named_references.ast'), 'all_tables', File.join(name,"formulae_no_table_references.ast")
+    replace ReplaceRangesWithArrayLiterals, File.join(name,"formulae_no_table_references.ast"), File.join(name,"formulae_no_ranges.ast")
+  end
+  
+  def replace_blanks
+    references = {}
+    worksheets("Loading formulae") do |name,xml_filename|
+      r = references[name] = {}
+      i = input(name,"formulae_no_indirects_optimised.ast")
+      i.lines do |line|
+        ref = line[/^(.*?)\t/,1]
+        r[ref] = true
+      end
+    end
+    worksheets("Replacing blanks") do |name,xml_filename|
+      #fork do 
+        r = ReplaceBlanks.new
+        r.references = references
+        r.default_sheet_name = name
+        replace r, File.join(name,"formulae_no_indirects_optimised.ast"),File.join(name,"formulae_no_blanks.ast")
+      #end
+    end
+  end
+  
+  def optimise_and_replace_indirect_loop
+    number_of_loops = 4
+    1.upto(number_of_loops) do |pass|
+      puts "Optimise and replace indirects pass #{pass}"
+      start = pass == 1 ? "formulae_no_ranges.ast" : "optimse-output-#{pass-1}.ast"
+      finish = pass == number_of_loops ? "formulae_no_indirects_optimised.ast" : "optimse-output-#{pass}.ast"
+      replace_indirects(start,"replace-indirect-output-#{pass}.ast","replace-indirect-working-#{pass}-")
+      optimise_sheets("replace-indirect-output-#{pass}.ast",finish,"optimse-working-#{pass}-")
+    end
+  end
+  
+  def replace_indirects(start_filename,finish_filename,basename)
+    worksheets("Replacing indirects") do |name,xml_filename|
+      counter = 1
+      replace ReplaceIndirectsWithReferences, File.join(name,start_filename), File.join(name,"#{basename}#{counter+1}.ast")
+      counter += 1
+
+      r = ReplaceNamedReferences.new
+      r.sheet_name = name
+      replace r, File.join(name,"#{basename}#{counter}.ast"), 'named_references.ast', File.join(name,"#{basename}#{counter+1}.ast")
+      counter += 1
+      
+      r = ReplaceTableReferences.new
+      r.sheet_name = name
+      replace r, File.join(name,"#{basename}#{counter}.ast"), 'all_tables', File.join(name,"#{basename}#{counter+1}.ast")
+      counter += 1
+      
+      replace ReplaceRangesWithArrayLiterals, File.join(name,"#{basename}#{counter}.ast"), File.join(name,"#{basename}#{counter+1}.ast")
+      counter += 1
+      replace ReplaceArraysWithSingleCells, File.join(name,"#{basename}#{counter}.ast"), File.join(name,"#{basename}#{counter+1}.ast")
+      counter += 1
+      
+      # Finally, create the output directory
+      i = File.join(intermediate_directory,name,"#{basename}#{counter}.ast")
+      o = File.join(intermediate_directory,name,finish_filename)
+      `cp '#{i}' '#{o}'`
+    end
+  end
+  
+  def optimise_sheets(start_filename,finish_filename,basename)
+    counter = 1
+    
+    # Setup start
+    worksheets("Setting up for optimise -#{counter}") do |name|
+      i = File.join(intermediate_directory,name,start_filename)
+      o = File.join(intermediate_directory,name,"#{basename}#{counter}.ast")
+      `cp '#{i}' '#{o}'`
+    end
+    
+    worksheets("Replacing with calculated values #{counter}-#{counter+1}") do |name,xml_filename|
+      #fork do
+        replace ReplaceFormulaeWithCalculatedValues, File.join(name,"#{basename}#{counter}.ast"), File.join(name,"#{basename}#{counter+1}.ast")
+      #end
+    end
+    counter += 1
+    Process.waitall
+      
+    references = all_formulae("#{basename}#{counter}.ast")
+    inline_ast_decision = lambda do |sheet,cell,references|
+      references_to_keep = @cells_that_can_be_set_at_runtime[sheet]
+      if references_to_keep && (references_to_keep == :all || references_to_keep.include?(cell))
+        false
+      else
+        ast = references[sheet][cell]
+        if ast
+          if [:number,:string,:blank,:null,:error,:boolean_true,:boolean_false,:sheet_reference,:cell].include?(ast.first)
+            #   puts "Inlining #{sheet}.#{cell}: #{ast.inspect}"
+            true
+          else
+            false
+          end
+        else
+          true # Always inline blanks
+        end
+      end
+    end
+    r = InlineFormulae.new
+    r.references = references
+    r.inline_ast = inline_ast_decision
+    
+    worksheets("Inlining formulae #{counter}-#{counter+1}") do |name,xml_filename|
+      #fork do
+        r.default_sheet_name = name
+        replace r, File.join(name,"#{basename}#{counter}.ast"), File.join(name,"#{basename}#{counter+1}.ast")
+      #end
+    end
+    counter += 1
+    Process.waitall
+    
+    # Finish
+    worksheets("Moving sheets #{counter}-") do |name|
+      o = File.join(intermediate_directory,name,finish_filename)
+      i = File.join(intermediate_directory,name,"#{basename}#{counter}.ast")
+      `cp '#{i}' '#{o}'`
+    end
+  end
+  
+  def remove_any_cells_not_needed_for_outputs(formula_in = "formulae_no_blanks.ast", formula_out = "formulae_pruned.ast", values_in = "values_no_shared_strings.ast", values_out = "values_pruned.ast")
+    if cells_to_keep && !cells_to_keep.empty?
+      identifier = IdentifyDependencies.new
+      identifier.references = all_formulae(formula_in)
+      cells_to_keep.each do |sheet_to_keep,cells_to_keep|
+        if cells_to_keep == :all
+          identifier.add_depedencies_for(sheet_to_keep)
+        elsif cells_to_keep.is_a?(Array)
+          cells_to_keep.each do |cell|
+            identifier.add_depedencies_for(sheet_to_keep,cell)
+          end
+        end
+      end
+      r = RemoveCells.new
+      worksheets("Removing cells") do |name,xml_filename|
+        #fork do
+          r.cells_to_keep = identifier.dependencies[name]
+          rewrite r, File.join(name, formula_in), File.join(name, formula_out)
+          rewrite r, File.join(name, values_in), File.join(name, values_out)
+        #end
+      end
+      Process.waitall
+    else
+      worksheets do |name,xml_filename|
+        i = File.join(intermediate_directory,name, formula_in)
+        o = File.join(intermediate_directory,name, formula_out)
+        `cp '#{i}' '#{o}'`
+        i = File.join(intermediate_directory,name, values_in)
+        o = File.join(intermediate_directory,name, values_out)
+        `cp '#{i}' '#{o}'`
+      end
+    end
+  end
+  
+  def inline_formulae_that_are_only_used_once
+    references = all_formulae("formulae_pruned.ast")
+    counter = CountFormulaReferences.new
+    count = counter.count(references)
+    
+    inline_ast_decision = lambda do |sheet,cell,references|
+      references_to_keep = @cells_that_can_be_set_at_runtime[sheet]
+      if references_to_keep && (references_to_keep == :all || references_to_keep.include?(cell))
+        false
+      else
+        count[sheet][cell] == 1
+      end
+    end
+    
+    r = InlineFormulae.new
+    r.references = references
+    r.inline_ast = inline_ast_decision
+    
+    worksheets("Inlining formulae") do |name,xml_filename|
+      #fork do
+        r.default_sheet_name = name
+        replace r, File.join(name,"formulae_pruned.ast"), File.join(name,"formulae_inlined.ast")
+      #end
+    end
+    
+    remove_any_cells_not_needed_for_outputs("formulae_inlined.ast", "formulae_inlined_pruned.ast", "values_pruned.ast", "values_pruned2.ast")
+
+  end
+  
+  def separate_formulae_elements
+    # First we add the sheet to all references, so that we can then look for common elements accross worksheets
+    r = RewriteCellReferencesToIncludeSheet.new
+    worksheets("Adding the sheet to all references") do |name,xml_filename|
+      r.worksheet = name
+      rewrite r, File.join(name,"formulae_inlined_pruned.ast"), File.join(name,"formulae_inlined_pruned_with_sheets.ast") 
+    end
+    
+    references = all_formulae("formulae_inlined_pruned_with_sheets.ast")
+    identifier = IdentifyRepeatedFormulaElements.new
+    repeated_elements = identifier.count(references)
+    repeated_elements.delete_if do |element,count|
+      count < 2
+    end
+    o = intermediate('common-elements-1.ast')
+    i = 0
+    repeated_elements.each do |element,count|
+      o.puts "common#{i}\t#{element}"
+      i = i + 1
+    end
+    close(o)
+    
+    worksheets("Replacing repeated elements") do |name,xml_filename|
+      replace ReplaceCommonElementsInFormulae, File.join(name,"formulae_inlined_pruned_with_sheets.ast"), "common-elements-1.ast", File.join(name,"formulae_inlined_pruned_replaced-1.ast")
+    end
+
+    r = ReplaceValuesWithConstants.new  
+    worksheets("Replacing values with constants") do |name,xml_filename|
+      i = input(name,"formulae_inlined_pruned_replaced-1.ast")
+      o = intermediate(name,"formulae_inlined_pruned_replaced.ast")
+      r.replace(i,o)
+      close(i,o)
+    end
+    
+    puts "Replacing values with constants in common elements"
+    i = input("common-elements-1.ast")
+    o = intermediate("common-elements.ast")
+    r.replace(i,o)
+    close(i,o)
+    
+    puts "Writing out constants"
+    co = intermediate("value_constants.ast")
+    r.rewriter.constants.each do |ast,constant|
+      co.puts "#{constant}\t#{ast}"
+    end
+    close(co)
+  end
+  
+  def write_out_excel_as_code
+    
+    all_refs = all_formulae("formulae_inlined_pruned_replaced.ast")
+    
+    number_of_refs = 0
+    
+    # Probably a better way of getting the runtime file to be compiled with the created file
+    puts `cp #{File.join(File.dirname(__FILE__),'..','compile','c','excel_to_c_runtime.c')} #{File.join(output_directory,'excel_to_c_runtime.c')}`
+    
+    # Output the workbook preamble
+    w = input("worksheet_c_names")
+    o = output("#{output_name.downcase}.c")
+    o.puts "// #{excel_file} approximately translated into C"
+    o.puts '#include "excel_to_c_runtime.c"'
+    o.puts
+    
+    # Now we have to put all the initial definitions out
+    o.puts "// definitions"
+
+    i = input("common-elements.ast")
+    c = CompileToCHeader.new
+    c.gettable = lambda { |ref| false }
+    c.rewrite(i,w,o)
+    i.rewind
+    number_of_refs += i.lines.to_a.size
+    close(i)
+
+    worksheets("Compiling definitions") do |name,xml_filename|
+      w.rewind
+      c = CompileToCHeader.new
+      c.settable = settable(name)
+      c.gettable = gettable(name)
+      c.worksheet = name
+      i = input(name,"formulae_inlined_pruned_replaced.ast")
+      c.rewrite(i,w,o)
+      i.rewind
+      number_of_refs += i.lines.to_a.size
+      close(i)
+    end
+    
+    o.puts "// end of definitions"
+    o.puts
+    o.puts "// Used to decide whether to recalculate a cell"
+    o.puts "static int variable_set[#{number_of_refs}];"
+    o.puts "void reset() {"
+    o.puts "  int i;"
+    o.puts "  cell_counter = 0;"
+    o.puts "  for(i = 0; i < #{number_of_refs}; i++) {"
+    o.puts "    variable_set[i] = 0;"
+    o.puts "  }"
+    o.puts "};"
+    o.puts
+    
+    # Output the value constants
+    o.puts "// starting the value constants"
+    mapper = MapValuesToCStructs.new
+    i = input("value_constants.ast")
+    i.lines do |line|
+      begin
+        ref, formula = line.split("\t")
+        ast = eval(formula)
+        calculation = mapper.map(ast)
+        o.puts "static ExcelValue #{ref} = #{calculation};"
+      rescue Exception => e
+        puts "Exception at line #{line}"
+        raise
+      end
+    end          
+    close(i)
+    o.puts "// ending the value constants"
+    o.puts
+    
+    variable_set_counter = 0
+    
+    # output the common elements
+    o.puts "// starting common elements"
+    w.rewind
+    c = CompileToC.new
+    c.variable_set_counter = variable_set_counter
+    c.gettable = lambda { |ref| false }
+    i = input("common-elements.ast")
+    c.rewrite(i,w,o)
+    close(i)
+    o.puts "// ending common elements"
+    o.puts
+    
+    variable_set_counter = c.variable_set_counter
+    
+    c = CompileToC.new
+    c.variable_set_counter = variable_set_counter
+    # Output the elements from each worksheet in turn
+    worksheets("Compiling worksheet") do |name,xml_filename|
+      w.rewind
+      c.settable = settable(name)
+      c.gettable = gettable(name)
+      c.worksheet = name
+
+      i = input(name,"formulae_inlined_pruned_replaced.ast")
+      ruby_name = c_name_for_worksheet_name(name)
+      o.puts "// start #{name}"
+      c.rewrite(i,w,o)
+      o.puts "// end #{name}"
+      o.puts
+      close(i)
+    end
+    close(w,o)
+  end
+  
+  def settable(name)
+    settable_refs = @cells_that_can_be_set_at_runtime[name]    
+    if settable_refs
+      lambda { |ref| (settable_refs == :all) ? true : settable_refs.include?(ref) } 
+    else
+      lambda { |ref| false }
+    end
+  end
+  
+  def gettable(name)
+    if @cells_to_keep
+      gettable_refs = @cells_to_keep[name]
+      if gettable_refs
+        lambda { |ref| (gettable_refs == :all) ? true : gettable_refs.include?(ref) }
+      else
+        lambda { |ref| false }
+      end
+    else
+      lambda { |ref| true }
+    end
+  end
+    
+  def write_build_script
+    o = output("Makefile")
+    name = output_name.downcase
+    
+    # Target for shared library
+    o.puts "lib#{name}.dylib: #{name}.o"
+    o.puts "\tgcc -shared -o lib#{name}.dylib #{name}.o"
+    o.puts
+    
+    # Target for compiled version
+    o.puts "#{name}.o:"
+    o.puts "\tgcc -Wall -fPIC -c #{name}.c"
+    o.puts
+    
+    # Target for cleaning
+    o.puts "clean:"
+    o.puts "\trm #{name}.o"
+    o.puts "\trm lib#{name}.dylib"
+    
+    close(o)
+  end
+  
+  def write_fuby_ffi_interface
+    all_formulae = all_formulae('formulae_inlined_pruned_replaced.ast')
+    name = output_name.downcase
+    o = output("#{name}.rb")
+    code = <<END
+require 'ffi'
+
+module #{name.capitalize}
+  extend FFI::Library
+  ffi_lib  File.join(File.dirname(__FILE__),'lib#{name}.dylib')
+  ExcelType = enum :ExcelEmpty, :ExcelNumber, :ExcelString, :ExcelBoolean, :ExcelError, :ExcelRange
+                
+  class ExcelValue < FFI::Struct
+    layout :type, ExcelType,
+  	       :number, :double,
+  	       :string, :string,
+         	 :array, :pointer,
+           :rows, :int,
+           :columns, :int             
+  end
+  
+END
+    o.puts code
+    o.puts
+    o.puts "  # use this function to reset all cell values"
+    o.puts "  attach_function 'reset', [], :void"
+
+    worksheets("Adding references to ruby shim for") do |name,xml_filename|
+      o.puts
+      o.puts "  # start of #{name}"  
+      c_name = c_name_for_worksheet_name(name)
+
+      # Put in place the setters, if any
+      settable_refs = @cells_that_can_be_set_at_runtime[name]
+      if settable_refs
+        settable_refs = all_formulae[name].keys if settable_refs == :all
+        settable_refs.each do |ref|
+          o.puts "  attach_function 'set_#{c_name}_#{ref.downcase}', [ExcelValue.by_value], :void"
+        end
+      end
+
+      if !cells_to_keep || cells_to_keep.empty? || cells_to_keep[name] == :all
+        getable_refs = all_formulae[name].keys
+      elsif !cells_to_keep[name] && settable_refs
+        getable_refs = settable_refs
+      else
+        getable_refs = cells_to_keep[name] || []
+      end
+        
+      getable_refs.each do |ref|
+        o.puts "  attach_function '#{c_name}_#{ref.downcase}', [], ExcelValue.by_value"
+      end
+        
+      o.puts "  # end of #{name}"
+    end
+    o.puts "end"  
+    close(o)
+  end
+  
+  def write_tests
+    name = output_name.downcase
+    o = output("#{name}_test.rb")    
+    o.puts "# coding: utf-8"
+    o.puts "# Test for #{name}"
+    o.puts "require 'rubygems'"
+    o.puts "gem 'minitest'"
+    o.puts  "require 'test/unit'"
+    o.puts  "require_relative '#{output_name.downcase}'"
+    o.puts
+    o.puts "class Test#{name.capitalize} < Test::Unit::TestCase"
+    o.puts "  def spreadsheet; @spreadsheet ||= init_spreadsheet; end"
+    o.puts "  def init_spreadsheet; #{name.capitalize} end"
+    
+    all_formulae = all_formulae('formulae_inlined_pruned_replaced.ast')
+    
+    worksheets("Adding tests for") do |name,xml_filename|
+      i = input(name,"values_pruned2.ast")
+      o.puts
+      o.puts "  # start of #{name}"  
+      c_name = c_name_for_worksheet_name(name)
+      if !cells_to_keep || cells_to_keep.empty? || cells_to_keep[name] == :all
+        refs_to_test = all_formulae[name].keys
+      else
+        refs_to_test = cells_to_keep[name]
+      end
+      if refs_to_test && !refs_to_test.empty?
+        CompileToCUnitTest.rewrite(i, c_name, refs_to_test, o)
+      end
+      close(i)
+    end
+    o.puts "end"
+    close(o)
+  end
+  
+  def compile_c_code
+    return unless actually_compile_c_code || actually_run_tests
+    puts "Compiling the resulting c code"
+    puts `cd #{File.join(output_directory)}; make clean; make`
+  end
+  
+  def run_tests
+    return unless actually_run_tests
+    puts "Running the resulting tests"
+    puts `cd #{File.join(output_directory)}; ruby "#{output_name.downcase}_test.rb"`
+  end
+  
+  
+  # UTILITY FUNCTIONS
+  
+  def all_formulae(filename)
+    references = {}
+    worksheets do |name,xml_filename|
+      r = references[name] = {}
+      i = input(name,filename)
+      i.lines do |line|
+        line =~ /^(.*?)\t(.*)$/
+        ref, ast = $1, $2
+        r[$1] = eval($2)
+      end
+    end 
+    references
+  end
+  
+  def c_name_for_worksheet_name(name)
+    unless @worksheet_names
+      w = input("worksheet_c_names")
+      @worksheet_names = Hash[w.readlines.map { |line| line.split("\t").map { |a| a.strip }}]
+      close(w)
+    end
+    @worksheet_names[name]
+  end
+    
+  def worksheets(message = "Processing",&block)
+    IO.readlines(File.join(intermediate_directory,'worksheet_names')).each do |line|
+      name, filename = *line.split("\t")
+      filename = File.expand_path(File.join(xml_directory,'xl',filename.strip))
+      puts "#{message} #{name}"
+      block.call(name, filename)
+    end
+  end
+    
+  def extract(_klass,xml_name,output_name)
+    i = xml_name.is_a?(String) ? xml(xml_name) : xml_name
+    o = output_name.is_a?(String) ? intermediate(output_name) : output_name
+    _klass.extract(i,o)
+    if xml_name.is_a?(String)
+      close(i)
+    end
+    if output_name.is_a?(String)
+      close(o)
+    end
+  end
+  
+  def rewrite(_klass,*args)
+    o = intermediate(args.pop)
+    inputs = args.map { |name| input(name) }
+    _klass.rewrite(*inputs,o)
+    close(*inputs,o)
+  end
+  
+  def replace(_klass,*args)
+    o = intermediate(args.pop)
+    inputs = args.map { |name| input(name) }
+    _klass.replace(*inputs,o)
+    close(*inputs,o)
+  end
+  
+  def xml(*args)
+    File.open(File.join(xml_directory,'xl',*args),'r')
+  end
+  
+  def input(*args)
+    File.open(File.join(intermediate_directory,*args),'r')
+  end
+  
+  def intermediate(*args)
+    File.open(File.join(intermediate_directory,*args),'w')
+  end
+  
+  def output(*args)
+    File.open(File.join(output_directory,*args),'w')
+  end
+  
+  def close(*args)
+    args.map(&:close)
+  end
+  
+end