excel_to_code 0.1.23 → 0.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 80514163db20835472c1af4fcd9d58c5705037cf
-  data.tar.gz: 50da754f9c831eb99917d0f460a6876db0c9c3b5
+  metadata.gz: 9c1bb313b18cf981e477100780e9f33bddc25eba
+  data.tar.gz: aee477a185842cb7feaf2efc0dfae1e3de81e7b9
 SHA512:
-  metadata.gz: d4c99e4ffbf3319fe1a95390ebd0ce149d543cd60a2c207955a925083bb9e68ae518a84d2bcbea429b67d7f63a87923dc18dde956bda74cda3503c1750987148
-  data.tar.gz: 1c2929b8d1474a63e085013545fd6a0250bc5863b1294bed9c61c8fa9a984176d92e298fdddf722b8e79c67423c3040ec8ef9e64c6e011277bba2aa251d89153
+  metadata.gz: 712f6b5fd56caa1a531d2a2c60355998b1b51e3a80594c92b906790d814f764f9447e671274c5f8cc97a7b2402f8c47b58f16f0c1b41c9b0d849f66984146646
+  data.tar.gz: 3e93b0e883b059728867256bd3c59dc63e73d869bb238e0aa6c7049175631b8fe60ef8d1639cbbc80ebf150cbf921015bdb7d78afab82a211116ea4bf26c34c3

data/src/commands/excel_to_c.rb

@@ -17,13 +17,11 @@ class ExcelToC < ExcelToX
   end
     
   def write_out_excel_as_code
+    log.info "Writing C code"
         
-    all_refs = all_formulae
-    
-    number_of_refs = 0
+    number_of_refs = @formulae.size
         
     # Output the workbook preamble
-    w = input('Worksheet C names')
     o = output("#{output_name.downcase}.c")
     o.puts "// #{excel_file} approximately translated into C"
 
@@ -38,29 +36,13 @@ class ExcelToC < ExcelToX
     o.puts "// definitions"
     o.puts "static ExcelValue ORIGINAL_EXCEL_FILENAME = {.type = ExcelString, .string = #{excel_file.inspect} };"
 
-    i = input("Common elements")
     c = CompileToCHeader.new
-    c.gettable = lambda { |ref| false }
-    c.rewrite(i,w,o)
-    i.rewind
-    number_of_refs += i.each_line.to_a.size
-    close(i)
-
-    worksheets do |name,xml_filename|
-      w.rewind
-      c = CompileToCHeader.new
-      c.settable = settable(name)
-      c.gettable = gettable(name)
-      c.worksheet = name
-      i = input([name,"Formulae"])
-      c.rewrite(i,w,o)
-      i.rewind
-      number_of_refs += i.each_line.to_a.size
-      close(i)
-    end
-    
+    c.settable = settable
+    c.gettable = gettable
+    c.rewrite(@formulae, @worksheet_c_names, o)
+
     # Need to make sure there are enough refs for named references as well
-    number_of_refs += named_references_to_keep.size
+    number_of_refs += @named_references_to_keep.size
 
     o.puts "// end of definitions"
     o.puts
@@ -82,85 +64,59 @@ class ExcelToC < ExcelToX
     # Output the value constants
     o.puts "// starting the value constants"
     mapper = MapValuesToCStructs.new
-    i = input("Constants")
-    i.each_line do |line|
+    @constants.each do |ref, ast|
       begin
-        ref, formula = line.split("\t")
-        ast = eval(formula)
         calculation = mapper.map(ast)
         o.puts "static ExcelValue #{ref} = #{calculation};"
       rescue Exception => e
-        puts "Exception at line #{line}"
+        puts "Exception at #{ref} #{ast}"
         raise
       end
     end          
-    close(i)
     o.puts "// ending the value constants"
     o.puts
     
     variable_set_counter = 0
     
-    # output the common elements
-    o.puts "// starting common elements"
-    w.rewind
-    c = CompileToC.new
-    c.variable_set_counter = variable_set_counter
-    c.gettable = lambda { |ref| false }
-    c.worksheet = ""
-    i = input("Common elements")
-    c.rewrite(i,w,o)
-    close(i)
-    o.puts "// ending common elements"
-    o.puts
-    
-    variable_set_counter = c.variable_set_counter
-    
     c = CompileToC.new
     c.variable_set_counter = variable_set_counter
     # Output the elements from each worksheet in turn
-    worksheets do |name,xml_filename|
-      w.rewind
-      c.settable = settable(name)
-      c.gettable = gettable(name)
-      c.worksheet = name
-
-      i = input([name,"Formulae"])
-      o.puts "// start #{name}"
-      c.rewrite(i,w,o)
-      o.puts "// end #{name}"
-      o.puts
-      close(i)
-    end
+    c.settable = settable
+    c.gettable = gettable
+    c.rewrite(@formulae, @worksheet_c_names, o)
 
     # Output the named references
 
     # Getters
     o.puts "// Start of named references"
-    i = input('Named references to keep')
-    w.rewind
     c.gettable = lambda { |ref| true }
     c.settable = lambda { |ref| false }
-    c.worksheet = nil
-    c.rewrite(i,w,o)
-    close(i)
+    named_references_ast = {}
+    @named_references_to_keep.each do |ref|
+      c_name = ref.is_a?(Array) ? c_name_for(ref) : ["", c_name_for(ref)]
+      named_references_ast[c_name] = @named_references[ref]
+    end
+
+    c.rewrite(named_references_ast, @worksheet_c_names, o)
 
     # Setters
-    i = input('Named references to set')
-    w.rewind # Worksheet C names
-    
     c = CompileNamedReferenceSetters.new
     c.cells_that_can_be_set_at_runtime = cells_that_can_be_set_at_runtime
-    c.rewrite(i,w,o)
-
-    close(i)
+    named_references_ast = {}
+    @named_references_that_can_be_set_at_runtime.each do |ref|
+      named_references_ast[c_name_for(ref)] = @named_references[ref]
+    end
+    c.rewrite(named_references_ast, @worksheet_c_names, o)
     o.puts "// End of named references"
 
-    close(w,o)
+    close(o)
   end
   
   # FIXME: Should make a Rakefile, especially in order to make sure the dynamic library name
   # is set properly
   def write_build_script
+    log.info "Writing Build script"
+
     o = output("Makefile")
     name = output_name.downcase
     
@@ -184,7 +140,8 @@ class ExcelToC < ExcelToX
   end
   
   def write_fuby_ffi_interface
-    all_formulae = all_formulae()
+    log.info "Writing ruby FFI code"
+
     name = output_name.downcase
     o = output("#{name}.rb")
       
@@ -310,9 +267,8 @@ END
     o.puts "  # use this function to reset all cell values"
     o.puts "  attach_function 'reset', [], :void"
 
-    worksheets do |name,xml_filename|
-      o.puts
-      o.puts "  # start of #{name}"  
+
+    worksheets do |name, xml_filename|
       c_name = c_name_for_worksheet_name(name)
 
       # Put in place the setters, if any
@@ -326,7 +282,7 @@ END
 
       # Put in place the getters
       if !cells_to_keep || cells_to_keep.empty? || cells_to_keep[name] == :all
-        getable_refs = all_formulae[name].keys
+        getable_refs = @formulae.keys.select { |ref| ref.first == name }.map { |ref| ref.last } 
       elsif !cells_to_keep[name] && settable_refs
         getable_refs = settable_refs
       else
@@ -340,25 +296,17 @@ END
       o.puts "  # end of #{name}"
     end
 
-    # Now put in place the getters and setters for the named references
     o.puts "  # Start of named references"
-
     # Getters
-    i = input('Named references to keep')
-    i.each_line do |line|
-      name = line.strip.split("\t").first
-      o.puts "  attach_function '#{name}', [], ExcelValue.by_value"
+    @named_references_to_keep.each do |name|
+      o.puts "  attach_function '#{c_name_for(name)}', [], ExcelValue.by_value"
     end
-    close(i)
 
     # Setters
-    i = input('Named references to set')
-    i.each_line do |line|
-      name = line.strip.split("\t").first
-      o.puts "  attach_function 'set_#{name}', [ExcelValue.by_value], :void"
+    @named_references_that_can_be_set_at_runtime.each do |name|
+      o.puts "  attach_function 'set_#{c_name_for(name)}', [ExcelValue.by_value], :void"
     end
 
-    close(i)
     o.puts "  # End of named references"
 
     o.puts "end"  
@@ -366,6 +314,8 @@ END
   end
   
   def write_tests
+    log.info "Writing tests" 
+
     name = output_name.downcase
     o = output("test_#{name}.rb")    
     o.puts "# coding: utf-8"
@@ -383,10 +333,7 @@ END
     o.puts "  def worksheet; @worksheet ||= init_spreadsheet; end"
     o.puts "  def init_spreadsheet; #{ruby_module_name}Shim.new end"
     
-    i = input("References to test")
-    CompileToCUnitTest.rewrite(i, sloppy_tests, o)
-    close(i)
-
+    CompileToCUnitTest.rewrite(Hash[@references_to_test_array], sloppy_tests, @worksheet_c_names, @constants, o)
     o.puts "end"
     close(o)
   end

data/src/commands/excel_to_ruby.rb

@@ -21,8 +21,8 @@ class ExcelToRuby < ExcelToX
   def write_out_excel_as_code
     log.info "Starting to write out code"
     
-    w = input('Worksheet C names')
     o = output("#{output_name.downcase}.rb")
+
     o.puts "# coding: utf-8"
     o.puts "# Compiled version of #{excel_file}"
     # FIXME: Should include the ruby files as part of the output, so don't have any dependencies
@@ -32,51 +32,26 @@ class ExcelToRuby < ExcelToX
     o.puts "  include ExcelFunctions"
     o.puts "  def original_excel_filename; #{excel_file.inspect}; end"
     
-    o.puts  
-    o.puts "  # Starting common elements"
-    log.info "Starting to write code for common elements"
     c = CompileToRuby.new
-    i = input("Common elements")
-    w.rewind    
-    c.rewrite(i,w,o)
-    o.puts "  # Ending common elements"
+    c.settable = settable
+
+    c.rewrite(@formulae, @worksheet_c_names, o)
     o.puts
-    close(i)
-    log.info "Finished writing code for common elements"
-    
-    d = intermediate('Defaults')
-    
-    worksheets do |name,xml_filename|
-      log.info "Starting to write code for worksheet #{name}"
-      c.settable = settable(name)
-      c.worksheet = name
-      i = input([name,"Formulae"])
-      w.rewind
-      o.puts "  # Start of #{name}"
-      c.rewrite(i,w,o,d)
-      o.puts "  # End of #{name}"
-      o.puts ""
-      close(i)
-      log.info "Finished writing code for worksheet #{name}"
-    end   
-     
-    close(d)
     
     log.info "Starting to write initializer"
     o.puts
     o.puts "  # starting initializer"
     o.puts "  def initialize"
-    d = input('Defaults')
-    d.each_line do |line|
+    d = c.defaults
+    d.each do |line|
       o.puts line
     end
     o.puts "  end"
     o.puts ""
-    close(d)
     log.info "Finished writing initializer"
               
     o.puts "end"
-    close(w,o)
+    close(o)
     log.info "Finished writing code"
   end
 
@@ -91,9 +66,8 @@ class ExcelToRuby < ExcelToX
     o.puts "class Test#{ruby_module_name} < Test::Unit::TestCase"
     o.puts "  def worksheet; @worksheet ||= #{ruby_module_name}.new; end"
 
-    i = input("References to test")
-    CompileToCUnitTest.rewrite(i, sloppy_tests, o)
-    close(i)
+    CompileToCUnitTest.rewrite(Hash[@references_to_test_array], sloppy_tests, @worksheet_c_names, @constants, o)
+
     o.puts "end"   
     close(o)
   end

data/src/commands/excel_to_x.rb

@@ -3,6 +3,10 @@ require 'fileutils'
 require 'logger'
 require_relative '../excel_to_code'
 
+# FIXME: Correct case for all worksheet references
+# FIXME: Correct case and $ stripping from all cell references
+# FIXME: Replacing with c compatible names everywhere
+
 # Used to throw normally fatal errors
 class ExcelToCodeException < Exception; end
 class VersionedFileNotFoundException < Exception; end
@@ -44,7 +48,7 @@ class ExcelToX
   # Each named reference then has a function in the resulting C code of the form
   # void set_named_reference_mangled_into_a_c_function(ExcelValue newValue)
   #
-  # By default, no named references are output
+  # By default no named references are output
   attr_accessor :named_references_that_can_be_set_at_runtime
   
   # Optional attribute. Specifies which cells must appear in the final generated code.
@@ -109,21 +113,41 @@ class ExcelToX
     
     self.cells_that_can_be_set_at_runtime ||= {}
     
-    # Make sure that all the cell names are downcase and don't have any $ in them
+    # Make sure that all the cell names are upcase symbols and don't have any $ in them
     if cells_that_can_be_set_at_runtime.is_a?(Hash)
+  
+      # Make sure the sheet names are symbols
+      cells_that_can_be_set_at_runtime.keys.each do |sheet|
+        next if sheet.is_a?(Symbol)
+        cells_that_can_be_set_at_runtime[sheet.to_sym] = cells_that_can_be_set_at_runtime.delete(sheet)
+      end
+
       cells_that_can_be_set_at_runtime.keys.each do |sheet|
         next unless cells_that_can_be_set_at_runtime[sheet].is_a?(Array)
-        cells_that_can_be_set_at_runtime[sheet] = cells_that_can_be_set_at_runtime[sheet].map { |reference| reference.gsub('$','').upcase }
+        cells_that_can_be_set_at_runtime[sheet] = cells_that_can_be_set_at_runtime[sheet].map { |reference| reference.gsub('$','').upcase.to_sym }
       end
     end
 
-    # Make sure that all the cell names are downcase and don't have any $ in them
+    # Make sure that all the cell names are upcase symbols and don't have any $ in them
     if cells_to_keep
+      cells_to_keep.keys.each do |sheet|
+        next if sheet.is_a?(Symbol)
+        cells_to_keep[sheet.to_sym] = cells_to_keep.delete(sheet)
+      end
+
       cells_to_keep.keys.each do |sheet|
         next unless cells_to_keep[sheet].is_a?(Array)
-        cells_to_keep[sheet] = cells_to_keep[sheet].map { |reference| reference.gsub('$','').upcase }
+        cells_to_keep[sheet] = cells_to_keep[sheet].map { |reference| reference.gsub('$','').upcase.to_sym }
       end
     end  
+
+    if named_references_to_keep.is_a?(Array)
+      named_references_to_keep.map! { |named_reference| named_reference.downcase.to_sym }
+    end
+
+    if named_references_that_can_be_set_at_runtime.is_a?(Array)
+      named_references_that_can_be_set_at_runtime.map! { |named_reference| named_reference.downcase.to_sym }
+    end
     
     # Make sure the relevant directories exist
     self.excel_file = File.expand_path(excel_file)
@@ -148,7 +172,6 @@ class ExcelToX
     # into a series of plain text files
     extract_data_from_workbook
     extract_data_from_worksheets
-    merge_table_files
     
     # This turns named references that are specified as getters and setters
     # into a series of required cell references
@@ -167,7 +190,7 @@ class ExcelToX
     # These perform a series of transformations to the information
     # with the intent of removing any redundant calculations
     # that are in the excel.
-    simplify_worksheets # Replacing shared strings and named references with their actual values, tidying arithmetic
+    simplify # Replacing shared strings and named references with their actual values, tidying arithmetic
 
     # In case this hasn't been set by the user
     if @cells_that_can_be_set_at_runtime.empty?
@@ -182,8 +205,8 @@ class ExcelToX
     filter_named_references
 
     replace_formulae_with_their_results
-    remove_any_cells_not_needed_for_outputs
     inline_formulae_that_are_only_used_once
+    remove_any_cells_not_needed_for_outputs
     separate_formulae_elements
     replace_values_with_constants
     create_sorted_references_to_test
@@ -191,17 +214,6 @@ class ExcelToX
     # This actually creates the code (implemented in subclasses)
     write_code
     
-    # clear some memory here, before trying to compile
-    if run_in_memory
-      @files = nil
-      @cells_to_keep = nil
-      @cells_that_can_be_set_at_runtime = nil
-      # now do garbage collection, because what we've just done will have freed a lot of memory
-      GC.enable
-      GC.start
-      # TODO I think there's still another 500MB that could be freed here, when compiling decc_model
-    end
-        
     # These compile and run the code version of the excel (implemented in subclasses)
     compile_code
     run_tests
@@ -232,20 +244,61 @@ class ExcelToX
     extract_shared_strings
     extract_named_references
     extract_worksheet_names
-    extract_dimensions_from_worksheets
   end
-  
-  # Excel keeps a central file of strings that appear in worksheet cells
+
+  # @shared_strings is an array of strings
   def extract_shared_strings
-    extract ExtractSharedStrings, 'sharedStrings.xml', 'Shared strings'
+    log.info "Extracting shared strings"
+    # Excel keeps a central file of strings that appear in worksheet cells
+    xml('sharedStrings.xml') do |i|
+      @shared_strings = ExtractSharedStrings.extract(i)
+    end
   end
   
   # Excel keeps a central list of named references. This includes those
   # that are local to a specific worksheet.
+  # They are put in a @named_references hash
+  # The hash value is the ast for the reference
+  # The hash key is either [sheet, name] or name
+  # Note that the sheet and the name are always stored lowercase
   def extract_named_references
-    extract ExtractNamedReferences, 'workbook.xml', 'Named references'
-    apply_rewrite RewriteFormulaeToAst, 'Named references'
-    replace ReplaceRangesWithArrayLiterals, 'Named references', 'Named references'
+    log.info "Extracting named references"
+    # First we get the references in raw form
+    xml('workbook.xml') do |i|
+      @named_references = ExtractNamedReferences.extract(i)
+    end
+    # Then we parse them
+    @named_references.each do |name, reference|
+      parsed = CachingFormulaParser.parse(reference)
+      if parsed
+        @named_references[name] = parsed
+      else
+        $stderr.puts "Named reference #{name} #{reference} not parsed"
+        exit
+      end
+    end
+    # Replace A$1:B2 with [A1, A2, B1, B2]
+    @replace_ranges_with_array_literals_replacer ||= ReplaceRangesWithArrayLiteralsAst.new
+
+    @named_references.each do |name, reference|
+      @named_references[name] = @replace_ranges_with_array_literals_replacer.map(reference)
+    end
+
+    # Now we need to check the user specified named references
+    if named_references_to_keep.is_a?(Array)
+      named_references_to_keep.each.with_index do |named_reference, i|
+        next if @named_references.has_key?(named_reference)
+        log.warn "Named reference '#{named_reference}' in named_references_to_keep has not been found in the spreadsheet"
+        named_references_to_keep[i] = nil
+      end.compact!
+    end
+    if named_references_that_can_be_set_at_runtime.is_a?(Array)
+      named_references_that_can_be_set_at_runtime.each.with_index do |named_reference, i|
+        next if @named_references.has_key?(named_reference)
+        log.warn "Named reference '#{named_reference}' in named_references_that_can_be_set_at_runtime has not been found in the spreadsheet"
+        named_references_that_can_be_set_at_runtime[i] = nil
+      end.compact!
+    end
   end
 
   # Excel keeps a list of worksheet names. To get the mapping between
@@ -253,202 +306,252 @@ class ExcelToX
   # relationships files. We also need to mangle the name into something
   # that will work ok as a filesystem or program name
   def extract_worksheet_names
-    extract ExtractWorksheetNames, 'workbook.xml', 'Worksheet names'
-    extract ExtractRelationships, File.join('_rels','workbook.xml.rels'), 'Workbook relationships'
-    rewrite RewriteWorksheetNames, 'Worksheet names', 'Workbook relationships', 'Worksheet names'
-    rewrite MapSheetNamesToCNames, 'Worksheet names', 'Worksheet C names'
-  end
+    log.info "Extracting worksheet names"
+    
+    worksheet_rids = {}
 
-  # We want a central list of the maximum extent of each worksheet
-  # so that we can convert column (e.g., C:F) and row (e.g., 13:18)
-  # references into equivalent area references (e.g., C1:F30)
-  def extract_dimensions_from_worksheets 
-    log.info "Starting to extract dimensions from worksheets"  
-    dimension_file = intermediate('Worksheet dimensions')
-    extractor = ExtractWorksheetDimensions.new
-    worksheets do |name, xml_filename|
-      log.info "Extracting dimensions for #{name}"
-      dimension_file.write name
-      dimension_file.write "\t"
-      
-      extractor.extract(xml(xml_filename), dimension_file)
-      close(xml_filename)
+    xml('workbook.xml') do |i|
+      worksheet_rids = ExtractWorksheetNames.extract(i) # {'worksheet_name' => 'rId3' ...}
+    end
+    
+    xml_for_rids = {}
+    xml('_rels','workbook.xml.rels') do |i|
+      xml_for_rids = ExtractRelationships.extract(i) #{ 'rId3' => "worlsheets/sheet1.xml" }
+    end
+
+    @worksheet_xmls = {}
+    worksheet_rids.each do |name, rid|
+      worksheet_xml = xml_for_rids[rid]
+      if worksheet_xml =~ /^worksheets/i # This gets rid of things that look like worksheets but aren't (e.g., chart sheets)
+        @worksheet_xmls[name.to_sym] = worksheet_xml
+      end
+    end
+    # FIXME: Extract this and put it at the end ?
+    @worksheet_c_names = {}
+    worksheet_rids.keys.each do |excel_worksheet_name|
+      @worksheet_c_names[excel_worksheet_name] = @worksheet_c_names[excel_worksheet_name.to_sym] = c_name_for(excel_worksheet_name)
     end
-    close(dimension_file)
   end
+
+  def c_name_for(name)
+    name = name.to_s
+    @c_names_assigned ||= {}
+    return @c_names_assigned.invert.fetch(name) if @c_names_assigned.has_value?(name)
+    c_name = name.downcase.gsub(/[^a-z0-9]+/,'_') # Make it lowercase, replace anything that isn't a-z or 0-9 with underscores
+    c_name = "s"+c_name if c_name[0] !~ /[a-z]/ # Can't start with a number. If it does, but an 's' in front (so 2010 -> s2010)
+    c_name = c_name + "2" if @c_names_assigned.has_key?(c_name) # Add a number at the end if the c_name has already been used
+    c_name.succ! while @c_names_assigned.has_key?(c_name)
+    @c_names_assigned[c_name] = name
+    c_name
+  end
+
   
-  # For each worksheet, this makes four passes through the xml
-  # 1. Extract the values of each cell
-  # 2. Extract all the cells which are simple formulae
-  # 3. Extract all the cells which use shared formulae
-  # 4. Extract all the cells which are part of array formulae
-  # 
-  # It then looks at the relationship file and extracts any tables
+  # For each worksheet, extract the useful bits from the excel xml
   def extract_data_from_worksheets
+    # All are hashes of the format ["SheetName", "A1"] => [:number, "1"]
+    # This one has a series of table references
+    extractor = ExtractEverythingFromWorkbook.new
+    
+    # Loop through the worksheets
+    # FIXME: make xml_filename be the IO object?
     worksheets do |name, xml_filename|
-      
-      extract ExtractValues, xml_filename, [name, 'Values']
-      apply_rewrite RewriteValuesToAst, [name, 'Values']
-      
-      extract ExtractSimpleFormulae, xml_filename, [name, 'Formulae (simple)']
-      apply_rewrite RewriteFormulaeToAst, [name, 'Formulae (simple)']
-
-      extract ExtractSharedFormulae, xml_filename, [name, 'Formulae (shared)']
-      apply_rewrite RewriteFormulaeToAst, [name, 'Formulae (shared)']
-
-      extract ExtractSharedFormulaeTargets, xml_filename, [name, 'Formulae (shared targets)']
-
-      extract ExtractArrayFormulae, xml_filename, [name, 'Formulae (array)']
-      apply_rewrite RewriteFormulaeToAst, [name, 'Formulae (array)']
-      
-      extract_tables_for_worksheet(name,xml_filename)
+      log.info "Extracting data from #{name}"
+      xml(xml_filename) do |input|
+        extractor.extract(name, input)
+      end
     end
+    @values = extractor.values
+    @formulae_simple = extractor.formulae_simple
+    @formulae_shared = extractor.formulae_shared
+    @formulae_shared_targets = extractor.formulae_shared_targets 
+    @formulae_array = extractor.formulae_array
+    @worksheets_dimensions = extractor.worksheets_dimensions
+    @table_rids = extractor.table_rids
+    @tables = {}
+    extract_tables
   end
   
   # To extract a table we need to look in the worksheet for table references
   # then we look in the relationships file for the filename that matches that
   # reference and contains the table data. Then we consolidate all the data
   # from individual table files into a single table file for the worksheet.
-  def extract_tables_for_worksheet(name, xml_filename)
-    extract ExtractWorksheetTableRelationships, xml_filename, [name, "Worksheet tables"]
-    extract ExtractRelationships, File.join('worksheets','_rels',"#{File.basename(xml_filename)}.rels"), [name, 'Relationships']
-    rewrite RewriteRelationshipIdToFilename, [name, "Worksheet tables"], [name, 'Relationships'], [name, "Worksheet tables"]
-    table_filenames = input(name, "Worksheet tables")
-    tables = intermediate(name, "Worksheet tables")
-    table_extractor = ExtractTable.new(name)
-    table_filenames.each_line do |line|
-      table_xml = xml(File.join('worksheets',line.strip))
-      table_extractor.extract(table_xml, tables)
-    end
-    close(tables,table_filenames)
-  end
-  
-  # Tables are like named references in that they can be referred to from
-  # anywhere in the workbook. Therefore we consolidate all the tables from
-  # all the worksheets into a central table file.
-  def merge_table_files
-    merged_table_file = intermediate("Workbook tables")
-    worksheets do |name,xml_filename|
-      log.info "Merging table files for #{name}"
-      worksheet_table_file = input([name, "Worksheet tables"])
-      worksheet_table_file.each_line do |line|
-        merged_table_file.puts line
+  def extract_tables
+    @table_rids.each do |worksheet_name, array_of_table_rids|
+      xml_filename = @worksheet_xmls[worksheet_name]
+      xml_for_rids = {}
+
+      # Load the relationship file
+      xml(File.join('worksheets','_rels',"#{File.basename(xml_filename)}.rels")) do |i|
+        xml_for_rids = ExtractRelationships.extract(i)
+      end
+      
+      # Then extract the individual tables
+      array_of_table_rids.each do |rid| 
+        xml(File.join('worksheets', xml_for_rids[rid])) do |i|
+          ExtractTable.extract(worksheet_name, i).each do |table_name, details|
+            @tables[table_name.downcase] = Table.new(table_name, *details)
+          end
+        end
       end
-      close worksheet_table_file
     end
-    close merged_table_file
   end
   
   def rewrite_worksheets
-    worksheets do |name,xml_filename|
-      log.info "Rewriting worksheet #{name}"
-      rewrite_row_and_column_references(name,xml_filename)
-      rewrite_shared_formulae(name,xml_filename)
-      rewrite_array_formulae(name,xml_filename)
-      combine_formulae_files(name,xml_filename)
-    end
+    rewrite_values
+    rewrite_row_and_column_references
+    rewrite_shared_formulae
+    rewrite_array_formulae
+    combine_formulae_files
+    simplify_arithmetic
   end
   
   # In Excel we can have references like A:Z and 5:20 which mean all cells in columns 
   # A to Z and all cells in rows 5 to 20 respectively. This function translates these
   # into more conventional references (e.g., A5:Z20) based on the maximum area that 
   # has been used on a worksheet
-  def rewrite_row_and_column_references(name,xml_filename)
-    dimensions = input('Worksheet dimensions')
-    
-    r = RewriteWholeRowColumnReferencesToAreas.new
-    r.worksheet_dimensions = dimensions
-    r.sheet_name = name
-    
-    apply_rewrite r, [name, 'Formulae (simple)']
-    apply_rewrite r, [name, 'Formulae (shared)']
-    apply_rewrite r, [name, 'Formulae (array)']
-    
-    dimensions.close
+  def rewrite_row_and_column_references
+    log.info "Rewriting row and column references"
+    # FIXME: Refactor
+    dimension_objects = {}
+    @worksheets_dimensions.map do |sheet_name, dimension| 
+      dimension_objects[sheet_name] = WorksheetDimension.new(dimension) 
+    end
+    mapper = MapColumnAndRowRangeAst.new(nil, dimension_objects)
+
+    @formulae_simple.each do |ref, ast|
+      mapper.default_worksheet_name = ref.first
+      mapper.map(ast)
+    end
+
+    @formulae_shared.each do |ref, ast|
+      mapper.default_worksheet_name = ref.first
+      mapper.map(ast.last)
+    end
+
+    @formulae_array.each do |ref, ast|
+      mapper.default_worksheet_name = ref.first
+      mapper.map(ast.last)
+    end
+    # FIXME: Could we now nil off the dimensions? Or do we need for indirects?
   end
   
-  def rewrite_shared_formulae(name,xml_filename)
-    rewrite RewriteSharedFormulae, [name, 'Formulae (shared)'], [name, 'Formulae (shared targets)'], [name, 'Formulae (shared)']
+  def rewrite_shared_formulae
+    log.info "Rewriting shared formulae"
+    @formulae_shared = RewriteSharedFormulae.rewrite( @formulae_shared, @formulae_shared_targets)
+    # FIXME: Could now nil off the @formula_shared_targets ?
   end
-  
-  def rewrite_array_formulae(name,xml_filename)
-    r = ReplaceNamedReferences.new
-    r.sheet_name = name
-    replace r, [name, 'Formulae (array)'], 'Named references', [name, 'Formulae (array)']
-
-    r = ReplaceTableReferences.new
-    r.sheet_name = name    
-    replace r,                              [name, 'Formulae (array)'], "Workbook tables", [name, 'Formulae (array)']
-    replace SimplifyArithmetic,             [name, 'Formulae (array)'], [name, 'Formulae (array)']
-    replace ReplaceRangesWithArrayLiterals, [name, 'Formulae (array)'], [name, 'Formulae (array)']
-    apply_rewrite RewriteArrayFormulaeToArrays,   [name, 'Formulae (array)']
-    apply_rewrite RewriteArrayFormulae,           [name, 'Formulae (array)']
+
+  def simplify_arithmetic
+    simplify_arithmetic_replacer ||= SimplifyArithmeticAst.new
+    @formulae.each do |ref, ast|
+      simplify_arithmetic_replacer.map(ast)
+    end
   end
   
-  def combine_formulae_files(name,xml_filename)
-    combiner = RewriteMergeFormulaeAndValues.new
-    combiner.references_to_add_if_they_are_not_already_present = required_references(name)
+  def rewrite_array_formulae
+    log.info "Rewriting array formulae"
+    # FIMXE: Refactor this
+
+    named_reference_replacer = ReplaceNamedReferencesAst.new(@named_references)
+    table_reference_replacer = ReplaceTableReferenceAst.new(@tables)
+    @replace_ranges_with_array_literals_replacer ||= ReplaceRangesWithArrayLiteralsAst.new
+    expand_array_formulae_replacer = AstExpandArrayFormulae.new
+    simplify_arithmetic_replacer ||= SimplifyArithmeticAst.new
+
+    # FIXME: THIS IS THE MOST HORRIFIC BODGE. I HATE IT.
+    @shared_string_replacer ||= ReplaceSharedStringAst.new(@shared_strings)
+    emergency_indirect_replacement_bodge = EmergencyArrayFormulaReplaceIndirectBodge.new
+    emergency_indirect_replacement_bodge.references = @values
     
-    rewrite combiner, [name, 'Values'], [name, 'Formulae (shared)'], [name, 'Formulae (array)'], [name, 'Formulae (simple)'], [name, 'Formulae']
+    @formulae_array.each do |ref, details|
+      @shared_string_replacer.map(details.last)
+      emergency_indirect_replacement_bodge.current_sheet_name = ref.first
+      emergency_indirect_replacement_bodge.replace(details.last)
+
+      named_reference_replacer.default_sheet_name = ref.first
+      named_reference_replacer.map(details.last)
+      table_reference_replacer.worksheet = ref.first
+      table_reference_replacer.referring_cell = ref.last
+      table_reference_replacer.map(details.last)
+      @replace_ranges_with_array_literals_replacer.map(details.last)
+      simplify_arithmetic_replacer.map(details.last)
+      expand_array_formulae_replacer.map(details.last)
+    end
+
+    @formulae_array = RewriteArrayFormulae.rewrite(@formulae_array)
+  end
+
+  def rewrite_values
+    log.info "Rewriting values"
+    r = ReplaceSharedStringAst.new(@shared_strings)
+    @values.each do |ref, ast|
+      r.map(ast)
+    end
+  end
+  
+  def combine_formulae_files
+    log.info "Combining formula files"
+
+    @formulae = required_references
+    # We dup this to avoid the values being replaced when manipulating formulae
+    @values.each do |ref, value|
+      @formulae[ref] = value.dup
+    end
+    @formulae.merge! @formulae_shared
+    @formulae.merge! @formulae_array
+    @formulae.merge! @formulae_simple
+
+    log.info "Sheet contains #{@formulae.size} cells"
   end
   
   # This ensures that all gettable and settable values appear in the output
   # even if they are blank in the underlying excel
-  def required_references(worksheet_name)
-    required_refs = []
-    if @cells_that_can_be_set_at_runtime && @cells_that_can_be_set_at_runtime[worksheet_name] && @cells_that_can_be_set_at_runtime[worksheet_name] != :all
-      required_refs.concat(@cells_that_can_be_set_at_runtime[worksheet_name])
+  def required_references
+    log.info "Checking required references"
+    required_refs = {}
+    if @cells_that_can_be_set_at_runtime && @cells_that_can_be_set_at_runtime != :named_references_only
+      @cells_that_can_be_set_at_runtime.each do |worksheet, refs|
+        next if refs == :all
+        refs.each do |ref|
+          required_refs[[worksheet, ref]] = [:blank]
+        end
+      end
     end
-    if @cells_to_keep && @cells_to_keep[worksheet_name] && @cells_to_keep[worksheet_name] != :all
-      required_refs.concat(@cells_to_keep[worksheet_name])
+    if @cells_to_keep
+      @cells_to_keep.each do |worksheet, refs|
+        next if refs == :all
+        refs.each do |ref|
+          required_refs[[worksheet, ref]] = [:blank]
+        end
+      end
     end
     required_refs
   end
 
-  # Returns a hash of named references, and the ast of their links
-  # where the named reference is global the key will be a string of
-  # its name and case sensitive.
-  # where the named reference is scoped to a worksheet, the key will be
-  # a two element array. The first element will be the sheet name. The
-  # second will be the name. 
-  def named_references
-    return @named_references if @named_references
-    @named_references = {}
-    i = input('Named references')
-    i.each_line do |line|
-      sheet, name, ref = *line.split("\t")
-      key = sheet.size != 0 ? [sheet, name] : name
-      @named_references[key] = eval(ref)
-    end
-    close(i)
-    @named_references
-  end
-
   # This makes sure that cells_to_keep includes named_references_to_keep
   def transfer_named_references_to_keep_into_cells_to_keep
-    log.debug "Started transfering named references to keep into cells to keep"
+    log.info "Transfering named references to keep into cells to keep"
     return unless @named_references_to_keep
-    @named_references_to_keep = named_references.keys if @named_references_to_keep == :all
+    @named_references_to_keep = @named_references.keys if @named_references_to_keep == :all
     @cells_to_keep ||= {}
-    all_named_references = named_references
     @named_references_to_keep.each do |name|
-      ref = all_named_references[name]
+      ref = @named_references[name]
       if ref
         add_ref_to_hash(ref, @cells_to_keep)
       else
-        log.warn "Named reference #{name} not found"
+        log.warn "Named reference "#{name}" not found"
       end
     end
   end
 
+  # This makes sure that there are cell setter methods for any named references that can be set
   def transfer_named_references_that_can_be_set_at_runtime_into_cells_that_can_be_set_at_runtime
-    log.debug "Started transfering named references that can be set at runtime into cells that can be set at runtime"
+    log.info "Making sure there are setter methods for named references that can be set"
     return unless @named_references_that_can_be_set_at_runtime
-    return if @named_references_that_can_be_set_at_runtime == :where_possible
+    return if @named_references_that_can_be_set_at_runtime == :where_possible # in this case will be done in #work_out_which_named_references_can_be_set_at_runtime
     @cells_that_can_be_set_at_runtime ||= {}
-    all_named_references = named_references
     @named_references_that_can_be_set_at_runtime.each do |name|
-      ref = all_named_references[name]
+      ref = @named_references[name]
       if ref
         add_ref_to_hash(ref, @cells_that_can_be_set_at_runtime)
       else
@@ -457,16 +560,21 @@ class ExcelToX
     end
   end
 
+  # The reference passed may be a sheet reference or an area reference
+  # in which case we need to expand out the ref so that the hash contains
+  # one reference per cell
   def add_ref_to_hash(ref, hash)
+    ref = ref.dup
     if ref.first == :sheet_reference
       sheet = ref[1]
-      cell = ref[2][1].gsub('$','')
+      cell = Reference.for(ref[2][1]).unfix.to_sym
       hash[sheet] ||= []
       return if hash[sheet] == :all
-      hash[sheet] << cell unless hash[sheet].include?(cell)
+      hash[sheet] << cell.to_sym unless hash[sheet].include?(cell.to_sym)
     elsif ref.first == :array
       ref.shift
       ref.each do |row|
+        row = row.dup
         row.shift
         row.each do |cell|
           add_ref_to_hash(cell, hash)
@@ -477,25 +585,33 @@ class ExcelToX
     end
   end
 
+  # This just checks which named references refer to cells that we have already declared as settable
   def work_out_which_named_references_can_be_set_at_runtime
+    log.info "Working out which named references can be set at runtime"
     return unless @named_references_that_can_be_set_at_runtime
     return unless @named_references_that_can_be_set_at_runtime == :where_possible
     cells_that_can_be_set = @cells_that_can_be_set_at_runtime
     cells_that_can_be_set = a_good_set_of_cells_that_should_be_settable_at_runtime if cells_that_can_be_set == :named_references_only
     cells_that_can_be_set_due_to_named_reference = Hash.new { |h,k| h[k] = Array.new  }
     @named_references_that_can_be_set_at_runtime = []
-    all_named_references = named_references
+    all_named_references = @named_references
+    # FIXME can this be refactored with #add_ref_to_hash
     @named_references_to_keep.each do |name|
       ref = all_named_references[name]
+      unless ref
+        log.warn "Named reference to keep #{name} not found in spreadsheet"
+        next
+      end
       if ref.first == :sheet_reference
         sheet = ref[1]
-        cell = ref[2][1].gsub('$','')
+        cell = Reference.for(ref[2][1]).unfix.to_sym
         s = cells_that_can_be_set[sheet]
         if s && s.include?(cell)
           @named_references_that_can_be_set_at_runtime << name 
-          cells_that_can_be_set_due_to_named_reference[sheet] << cell
+          cells_that_can_be_set_due_to_named_reference[sheet] << cell.to_sym
           cells_that_can_be_set_due_to_named_reference[sheet].uniq!
         end
+        #FIXME: Is this righ?
       elsif ref.first.is_a?(Array)
         ref = ref.first
         settable = ref.all? do |r|
@@ -509,7 +625,7 @@ class ExcelToX
           ref.each do |r| 
             sheet = r[1]
             cell = r[2][1].gsub('$','')
-            cells_that_can_be_set_due_to_named_reference[sheet] << cell
+            cells_that_can_be_set_due_to_named_reference[sheet] << cell.to_sym
             cells_that_can_be_set_due_to_named_reference[sheet].uniq!
           end
         end
@@ -521,129 +637,70 @@ class ExcelToX
   end
 
   # FIXME: Feels like a kludge
+  # This works out which named references should appear in the generated code
   def filter_named_references
+    log.info "Filtering named references to keep"
     @named_references_to_keep ||= []
     @named_references_that_can_be_set_at_runtime ||= []
 
-    i = input('Named references')
-    o = intermediate('Named references to keep')
-    i.each_line do |line|
-      sheet, name, ref = *line.split("\t")
-      key = sheet.length != 0 ? [sheet, name] : name
-      o.puts line if named_references_to_keep.include?(key) || named_references_that_can_be_set_at_runtime.include?(key)
-    end
-    close(o)
-
-    i.rewind
-    o = intermediate('Named references to set')
-    i.each_line do |line|
-      sheet, name, ref = *line.split("\t")
-      key = sheet.length != 0 ? [sheet, name] : name
-      o.puts line if named_references_that_can_be_set_at_runtime.include?(key)
+    @named_references.each do |name, ref|
+      if named_references_to_keep.include?(name) || named_references_that_can_be_set_at_runtime.include?(name)
+        # FIXME: Refactor the c_name_for to closer to the writing?
+        @named_references_to_keep << name
+      end
     end
-    close(o)
 
-    # FIXME: Might result in getter and setter having different names
-    rewrite RewriteNamedReferenceNames, 'Named references to keep', 'Worksheet C names', 'Named references to keep'
-    rewrite RewriteNamedReferenceNames, 'Named references to set', 'Worksheet C names', 'Named references to set'
-  end
-    
-  def simplify_worksheets
-    worksheets do |name,xml_filename|
-      replace ReplaceSharedStrings, [name, 'Values'], 'Shared strings', File.join(name, 'Values')
-      
-      replace SimplifyArithmetic,   [name, 'Formulae'], [name, 'Formulae']      
-      replace ReplaceSharedStrings, [name, 'Formulae'], 'Shared strings', [name, 'Formulae']
-      
-      r = ReplaceNamedReferences.new
-      r.sheet_name = name
-      replace r, [name, 'Formulae'], 'Named references', [name, 'Formulae']
-
-      r = ReplaceTableReferences.new
-      r.sheet_name = name
-      replace r, [name, 'Formulae'], "Workbook tables", [name, 'Formulae']
-      
-      replace ReplaceRangesWithArrayLiterals, [name, 'Formulae'],  [name, 'Formulae']
-      replace ReplaceArithmeticOnRanges, [name, 'Formulae'],  [name, 'Formulae']
-      replace ReplaceArraysWithSingleCells, [name, 'Formulae'],  [name, 'Formulae']
-      replace WrapFormulaeThatReturnArraysAndAReNotInArrays, [name, 'Formulae'],  [name, 'Formulae']
+    @named_references.each do |name, ref|
+      if named_references_that_can_be_set_at_runtime.include?(name)
+        @named_references_that_can_be_set_at_runtime << name
+      end
     end
   end
     
-  def replace_formulae_with_their_results
-    number_of_passes = 0
-    begin 
-      number_of_passes += 1
-      @replacements_made_in_the_last_pass = 0
-      replace_indirects_and_offsets
-      replace_formulae_with_calculated_values
-      replace_references_to_values_with_values
-      log.info "Pass #{number_of_passes}: Made #{@replacements_made_in_the_last_pass} replacements"
-      if number_of_passes > 20
-        log.warn "Made more than 20 passes, so aborting"
-        break
-      end
-    end while @replacements_made_in_the_last_pass > 0
-  end
-  
-  # There is no support for INDIRECT or OFFSET in the ruby or c runtime
-  # However, in many cases it isn't needed, because we can work
-  # out the value of the indirect or OFFSET at compile time and eliminate it
-  def replace_indirects_and_offsets
-    worksheets do |name,xml_filename|
-      log.info "Replacing INDIRECT, OFFSET and COLUMN functions in #{name}"
-      
-      # First of all we replace any indirects where their values can be calculated at compile time with those
-      # calculated values (e.g., INDIRECT("A"&1) can be turned into A1 and OFFSET(A1,1,1,2,2) can be turned into B2:C3)
-      [ReplaceIndirectsWithReferences.new, ReplaceOffsetsWithReferences.new, ReplaceColumnWithColumnNumber.new].each do |r|
-        replace r, [name, 'Formulae'],  [name, 'Formulae']
-        @replacements_made_in_the_last_pass += r.replacements_made_in_the_last_pass
-      end
-      
-      # The result of the indirect might be a named reference, which we need to simplify
-      r = ReplaceNamedReferences.new
-      r.sheet_name = name
-      replace r, [name, 'Formulae'], 'Named references', [name, 'Formulae']
-      
-      # The result of the indirect might contain arithmetic, which we need to simplify
-      replace SimplifyArithmetic,   [name, 'Formulae'], [name, 'Formulae']      
+  def simplify(cells = @formulae)
+    log.info "Simplifying cells"
 
-      # The result of the indirect might be a table reference, which we need to simplify
-      r = ReplaceTableReferences.new
-      r.sheet_name = name
-      replace r, [name, 'Formulae'], "Workbook tables", [name, 'Formulae']
-      
-      # The result of the indirect might be a range, which we need to simplify
-      replace ReplaceRangesWithArrayLiterals, [name, 'Formulae'],  [name, 'Formulae']
-      replace ReplaceArithmeticOnRanges, [name, 'Formulae'],  [name, 'Formulae']
-      replace ReplaceStringJoinOnRanges, [name, 'Formulae'],  [name, 'Formulae']
-      replace ReplaceArraysWithSingleCells, [name, 'Formulae'],  [name, 'Formulae']
-      replace WrapFormulaeThatReturnArraysAndAReNotInArrays, [name, 'Formulae'],  [name, 'Formulae']
-    end
-  end
-  
-  # If a formula's value can be calculated at compile time, it is replaced with its calculated value (e.g., 1+1 gets replaced with 2)
-  def replace_formulae_with_calculated_values    
-    worksheets do |name,xml_filename|
-      r = ReplaceFormulaeWithCalculatedValues.new
-      r.excel_file = excel_file
-      replace r, [name, 'Formulae'],  [name, 'Formulae']
-      @replacements_made_in_the_last_pass += r.replacements_made_in_the_last_pass
+    @shared_string_replacer ||= ReplaceSharedStringAst.new(@shared_strings)
+    @replace_arithmetic_on_ranges_replacer ||= ReplaceArithmeticOnRangesAst.new
+    @wrap_formulae_that_return_arrays_replacer ||= WrapFormulaeThatReturnArraysAndAReNotInArraysAst.new
+    @named_reference_replacer ||= ReplaceNamedReferencesAst.new(@named_references) 
+    @table_reference_replacer ||= ReplaceTableReferenceAst.new(@tables)
+    @replace_ranges_with_array_literals_replacer ||= ReplaceRangesWithArrayLiteralsAst.new
+    @replace_arrays_with_single_cells_replacer ||= ReplaceArraysWithSingleCellsAst.new
+    @replace_string_joins_on_ranges_replacer ||= ReplaceStringJoinOnRangesAST.new
+    @sheetless_cell_reference_replacer ||= RewriteCellReferencesToIncludeSheetAst.new
+
+    cells.each do |ref, ast|
+      @sheetless_cell_reference_replacer.worksheet = ref.first
+      @sheetless_cell_reference_replacer.map(ast)
+      @shared_string_replacer.map(ast)
+      @named_reference_replacer.default_sheet_name = ref.first
+      @named_reference_replacer.map(ast)
+      @table_reference_replacer.worksheet = ref.first
+      @table_reference_replacer.referring_cell = ref.last
+      @table_reference_replacer.map(ast)
+      @replace_ranges_with_array_literals_replacer.map(ast)
+      @replace_arithmetic_on_ranges_replacer.map(ast)
+      @replace_arrays_with_single_cells_replacer.map(ast)
+      @replace_string_joins_on_ranges_replacer.map(ast)
+      @wrap_formulae_that_return_arrays_replacer.map(ast)
     end
+
   end
 
-  # If a formula references a cell containing a value, the reference is replaced with the value (e.g., if A1 := 2 and A2 := A1 + 1 then becomes: A2 := 2 + 1)
-  def replace_references_to_values_with_values
-    references = all_formulae
-    
-    inline_ast_decision = lambda do |sheet,cell,references|
+  # These types of cells don't conatain formulae and can therefore be skipped
+  VALUE_TYPE = {:number => true, :string => true, :blank => true, :null => true, :error => true, :boolean_true => true, :boolean_false => true}
+  INLINE_TYPE = {:number => true, :string => true, :blank => true, :null => true, :error => true, :boolean_true => true, :boolean_false => true, :sheet_reference => true, :cell => true}
+  
+  def inline_ast_decision  
+    @inline_ast_decision ||= lambda do |sheet, cell, references|
       references_to_keep = @cells_that_can_be_set_at_runtime[sheet]
       if references_to_keep && (references_to_keep == :all || references_to_keep.include?(cell))
         false
       else
-        ast = references[sheet][cell]
+        ast = references[[sheet,cell]]
         if ast
-          if [:number,:string,:blank,:null,:error,:boolean_true,:boolean_false,:sheet_reference,:cell].include?(ast.first)
+          if INLINE_TYPE[ast.first]
             true
           else
             false
@@ -653,21 +710,81 @@ class ExcelToX
         end
       end
     end
-    
-    r = InlineFormulae.new
-    r.references = references
-    r.inline_ast = inline_ast_decision
-    
-    worksheets do |name,xml_filename|
-      r.default_sheet_name = name
-      replace r, [name, 'Formulae'],  [name, 'Formulae']
-      @replacements_made_in_the_last_pass += r.replacements_made_in_the_last_pass
+  end
+
+  def replace_formulae_with_their_results
+    number_of_passes = 0
+
+    @cells_with_formulae = @formulae.dup
+    @cells_with_formulae.each do |ref, ast|
+      @cells_with_formulae.delete(ref) if VALUE_TYPE[ast[0]]
     end
+
+    # Set up for replacing references to cells with the cell
+    inline_replacer = InlineFormulaeAst.new
+    inline_replacer.references = @formulae
+    inline_replacer.inline_ast = inline_ast_decision
+
+    value_replacer = MapFormulaeToValues.new
+    value_replacer.original_excel_filename = excel_file
+
+    # There is no support for INDIRECT or OFFSET in the ruby or c runtime
+    # However, in many cases it isn't needed, because we can work
+    # out the value of the indirect or OFFSET at compile time and eliminate it
+    # First of all we replace any indirects where their values can be calculated at compile time with those
+    # calculated values (e.g., INDIRECT("A"&1) can be turned into A1 and OFFSET(A1,1,1,2,2) can be turned into B2:C3)
+    indirect_replacement = ReplaceIndirectsWithReferencesAst.new
+    column_replacement = ReplaceColumnWithColumnNumberAST.new
+    offset_replacement = ReplaceOffsetsWithReferencesAst.new
+
+    begin 
+      number_of_passes += 1
+      log.info "Starting pass #{number_of_passes} on #{@cells_with_formulae.size} cells"
+
+      replacements_made_in_the_last_pass = 0
+      inline_replacer.count_replaced = 0
+      value_replacer.replacements_made_in_the_last_pass = 0
+      column_replacement.count_replaced = 0
+      offset_replacement.count_replaced = 0
+      indirect_replacement.count_replaced = 0
+      references_that_need_updating = {}
+
+      @cells_with_formulae.each do |ref, ast|
+        # FIXME: Shouldn't need to wrap ref.fist in an array
+        inline_replacer.current_sheet_name = [ref.first]
+        inline_replacer.map(ast)
+        # If a formula references a cell containing a value, the reference is replaced with the value (e.g., if A1 := 2 and A2 := A1 + 1 then becomes: A2 := 2 + 1)
+        value_replacer.map(ast)
+        if column_replacement.replace(ast)
+          references_that_need_updating[ref] = ast
+        end
+        if offset_replacement.replace(ast)
+          references_that_need_updating[ref] = ast
+        end
+        if indirect_replacement.replace(ast)
+          references_that_need_updating[ref] = ast
+        end
+        @cells_with_formulae.delete(ref) if VALUE_TYPE[ast[0]]
+      end
+
+      simplify(references_that_need_updating)
+
+      replacements_made_in_the_last_pass += inline_replacer.count_replaced
+      replacements_made_in_the_last_pass += value_replacer.replacements_made_in_the_last_pass
+      replacements_made_in_the_last_pass += column_replacement.count_replaced
+      replacements_made_in_the_last_pass += offset_replacement.count_replaced
+      replacements_made_in_the_last_pass += indirect_replacement.count_replaced
+
+      log.info "Pass #{number_of_passes}: Made #{replacements_made_in_the_last_pass} replacements"
+    end while replacements_made_in_the_last_pass > 0 && number_of_passes < 20
   end
+
+  
   
   # If 'cells to keep' are specified, then other cells are removed, unless
   # they are required to calculate the value of a cell in 'cells to keep'.
   def remove_any_cells_not_needed_for_outputs
+    log.info "Removing cells not needed for outputs"
 
     # If 'cells to keep' isn't specified, then ALL cells are kept
     return unless cells_to_keep && !cells_to_keep.empty?
@@ -675,7 +792,7 @@ class ExcelToX
     # Work out what cells the cells in 'cells to keep' need 
     # in order to be able to calculate their values
     identifier = IdentifyDependencies.new
-    identifier.references = all_formulae
+    identifier.references = @formulae
     cells_to_keep.each do |sheet_to_keep,cells_to_keep|
       if cells_to_keep == :all
         identifier.add_depedencies_for(sheet_to_keep)
@@ -701,22 +818,22 @@ class ExcelToX
     end
     
     # Now we actually go ahead and remove the cells
-    worksheets do |name,xml_filename|
-      r = RemoveCells.new
-      r.cells_to_keep = identifier.dependencies[name]
-      rewrite r, [name, 'Formulae'],  [name, 'Formulae']
-      rewrite r, [name, 'Values'],  [name, 'Values'] # Must remove the values as well, to avoid any tests being generated for cells that don't exist
-    end
+    r = RemoveCells.new
+    r.cells_to_keep = identifier.dependencies
+    r.rewrite(@formulae)
+    # Must remove the values as well, to avoid any tests being generated for cells that don't exist
+    r.rewrite(@values)
+    r.rewrite(@cells_with_formulae)
   end
   
   # If a cell is only referenced from one other cell, then it is inlined into that other cell
   # e.g., A1 := B3+B6 ; B1 := A1 + B3 becomes: B1 := (B3 + B6) + B3. A1 is removed.
   def inline_formulae_that_are_only_used_once
-    references = all_formulae
-    
+    log.info "Inlining formulae"
+
     # First step is to calculate how many times each cell is referenced by another cell
     counter = CountFormulaReferences.new
-    count = counter.count(references)
+    count = counter.count(@formulae)
     
     # This takes the decision:
     # 1. If a cell is in the list of cells to keep, then it is never inlined
@@ -726,22 +843,17 @@ class ExcelToX
       if references_to_keep && (references_to_keep == :all || references_to_keep.include?(cell))
         false
       else
-        count[sheet][cell] == 1
+        count[[sheet,cell]] == 1 # i.e., inline if used only once
       end
     end
     
-    r = InlineFormulae.new
-    r.references = references
+    r = InlineFormulaeAst.new
+    r.references = @formulae
     r.inline_ast = inline_ast_decision
-    
-    worksheets do |name,xml_filename|
-      r.default_sheet_name = name
-      replace r, [name, 'Formulae'],  [name, 'Formulae']
+    @cells_with_formulae.each do |ref, ast|
+      r.current_sheet_name = [ref.first]
+      r.map(ast)
     end
-    
-    # We need to do this again, to get rid of the cells that we have just inlined
-    # FIXME: This could be done more efficiently, given we know which cells were removed
-    remove_any_cells_not_needed_for_outputs
   end
   
   # This comes up with a list of references to test, in the form of a file called 'References to test'.
@@ -750,54 +862,42 @@ class ExcelToX
   # These will be sorted so that later refs depend on earlier refs. This should mean that the first test that 
   # fails will be the root cause of the problem
   def create_sorted_references_to_test
-    all_formulae = all_formulae()
+    log.info "Creating references to test"
+
     references_to_test = {}
 
     # First get the list of references we should test
-    worksheets do |name, xml_filename|
-      log.info "Workingout references to test for #{name}"
-
-      # Either keep all the cells on the sheet  
-      if !cells_to_keep || cells_to_keep.empty? || cells_to_keep[name] == :all
-        keep = all_formulae[name].keys || []
-      else # Or just those specified as cells that will be kept
-        keep = cells_to_keep[name] || []
-      end
-
-      # Now go through and match the cells to keep with their values
-      i = input([name,"Values"])
-      i.each_line do |line|
-        ref, formula = line.split("\t")
-        next unless keep.include?(ref.upcase)
-        references_to_test[[name, ref]] = formula
+    @values.each do |ref, value|
+      if !cells_to_keep || 
+          cells_to_keep.empty? || 
+          (cells_to_keep[ref.first] && (
+            cells_to_keep[ref.first] == :all ||
+            cells_to_keep[ref.first].include?(ref.last)
+          ))
+        references_to_test[ref] = value
       end
-      close(i)
     end
-    
+
     # Now work out dependency tree
-    sorted_references = SortIntoCalculationOrder.new.sort(all_formulae)
+    sorted_references = @formulae.keys #SortIntoCalculationOrder.new.sort(@formulae)
 
-    references_to_test_file = intermediate("References to test")
+    @references_to_test_array = []
     sorted_references.each do |ref|
-      ast = references_to_test[ref]
-      next unless ast
-      c_name = c_name_for_worksheet_name(ref[0])
-      references_to_test_file.puts "#{c_name}\t#{ref[1]}\t#{ast}"
+      next unless references_to_test.include?(ref)
+      @references_to_test_array << [ref, @values[ref]]
     end
-
-    close references_to_test_file
+    # FIXME: CNAMES
   end
 
 
   # This looks for repeated formula parts, and separates them out. It is the opposite of inlining:
   # e.g., A1 := (B1 + B3) + B10; A2 := (B1 + B3) + 3 gets transformed to: Common1 := B1 + B3 ; A1 := Common1 + B10 ; A2 := Common1 + 3
   def separate_formulae_elements
+    log.info "Looking for repeated bits of formulae"
     
-    replace_all_simple_references_with_sheet_references # So we can be sure which references are repeating and which references are distinct
     
-    references = all_formulae
     identifier = IdentifyRepeatedFormulaElements.new
-    repeated_elements = identifier.count(references)
+    repeated_elements = identifier.count(@cells_with_formulae)
     
     # We apply a threshold that something needs to be used twice for us to bother separating it out. 
     # FIXME: This threshold is arbitrary
@@ -805,216 +905,134 @@ class ExcelToX
       count < 2
     end
     
-    # Dump our selected common elements into a separate file of formulae
-    o = intermediate('Common elements')
-    i = 0
-    repeated_elements.each do |element,count|
-      o.puts "common#{i}\t#{element}"
-      i = i + 1
+    # Translate the repeated elements into a code of the form [:cell, "common#{1}"]
+    index = 0
+    repeated_element_ast = {}
+    repeated_elements.each do |ast, count|
+      repeated_element_ast[ast.dup] = [:cell, "common#{index}"]
+      index +=1 
     end
-    close(o)
-    
-    # Replace common elements in formulae with references to otherw
-    worksheets do |name,xml_filename|
-      replace ReplaceCommonElementsInFormulae, [name, 'Formulae'], "Common elements", [name, 'Formulae']
+
+    r = ReplaceCommonElementsInFormulae.new
+    r.replace(@cells_with_formulae, repeated_element_ast)
+    common_elements_used = r.common_elements_used
+
+    repeated_element_ast.delete_if do |repeated_ast, common_ast|
+      common_elements_used[common_ast] == 0
     end
-    # FIXME: This means that some common elements won't ever be called, becuase they are replaced by a longer common element
-    # Should the common elements be merged first?
-  end
 
-  # We add the sheet name to all references, so that we can then look for common elements accross worksheets
-  # e.g., A1 := A2 gets transformed to A1 := Sheet1!A2  
-  def replace_all_simple_references_with_sheet_references
-    r = RewriteCellReferencesToIncludeSheet.new
-    worksheets do |name,xml_filename|
-      r.worksheet = name
-      rewrite r, [name, 'Formulae'],  [name, 'Formulae']
+    # FIXME: Is this best? Seems to work
+    repeated_element_ast.each do |repeated_ast, common_ast|
+      @formulae[["", common_ast[1]]] = repeated_ast
     end
-  end  
+
+  end
   
   # This puts back in an optimisation that excel carries out by making sure that
   # two copies of the same value actually refer to the same underlying spot in memory
   def replace_values_with_constants
+    log.info "Replacing values with constants"
     
     # First do it in the formulae
-    r = ReplaceValuesWithConstants.new
-    worksheets do |name,xml_filename|
-      replace r, [name, 'Formulae'],  [name, 'Formulae']
+    r = MapValuesToConstants.new
+    @formulae.each do |ref, ast|
+      r.map(ast)
     end
-    
-    # Then do it in the common elements
-    replace r, "Common elements", "Common elements"
-    
-    # Then write out the constants
-    output = intermediate("Constants")
-    # FIXME: This looks bad!
-    r.rewriter.constants.each do |ast,constant|
-      output.puts "#{constant}\t#{ast}"
-    end
-    close(output)
+
+    @constants = r.constants.invert
   end
   
-  # If nothing has been specified in named_refernces_that_can_be_set_at_runtime 
+  # If nothing has been specified in named_references_that_can_be_set_at_runtime 
   # or in cells_that_can_be_set_at_runtime, then we assume that
   # all value cells should be settable if they are referenced by
   # any other forumla.
   def a_good_set_of_cells_that_should_be_settable_at_runtime
-    references = all_formulae
+    log.info "Generating a good set of cells that should be settable"
+
     counter = CountFormulaReferences.new
-    count = counter.count(references)
+    count = counter.count(@formulae)
     settable_cells = {}
+    settable_types = [:blank,:number,:null,:string,:shared_string,:constant,:percentage,:error,:boolean_true,:boolean_false]
 
-    count.each do |sheet,keys|
-      keys.each do |ref,count|
-        next unless count >= 1
-        next unless references[sheet]
-        ast = references[sheet][ref]
-        next unless ast
-        if [:blank,:number,:null,:string,:shared_string,:constant,:percentage,:error,:boolean_true,:boolean_false].include?(ast.first)
-          settable_cells[sheet] ||= []
-          settable_cells[sheet] << ref.upcase
-        end
-      end
+    count.each do |ref,count|
+      next unless count >= 1 # No point making a cell that isn't reference settable
+      ast = @formulae[ref]
+      next unless ast # Sometimes empty cells are referenced. 
+      next unless settable_types.include?(ast.first)
+      settable_cells[ref.first] ||= []
+      settable_cells[ref.first] << ref.last.upcase
     end
     return settable_cells
   end
   
   # UTILITY FUNCTIONS
 
-  def settable(name)
-    settable_refs = @cells_that_can_be_set_at_runtime[name]    
+  def settable
+    settable_refs = @cells_that_can_be_set_at_runtime
     if settable_refs
-      lambda { |ref| (settable_refs == :all) ? true : settable_refs.include?(ref.upcase) } 
+      lambda { |ref| 
+        sheet = ref.first
+        cell = ref.last
+        if settable_refs[sheet]
+          if settable_refs[sheet] == :all || settable_refs[sheet].include?(cell.upcase)
+            true
+          else
+            false
+          end
+        else
+          false
+        end
+      }
     else
       lambda { |ref| false }
     end
   end
   
-  def gettable(name)
+  def gettable
     if @cells_to_keep
-      gettable_refs = @cells_to_keep[name]
-      if gettable_refs
-        lambda { |ref| (gettable_refs == :all) ? true : gettable_refs.include?(ref.upcase) }
-      else
-        lambda { |ref| false }
-      end
+      gettable_refs = @cells_to_keep
+      lambda { |ref| 
+        sheet = ref.first
+        cell = ref.last
+        if gettable_refs[sheet]
+          if gettable_refs[sheet] == :all || gettable_refs[sheet].include?(cell.upcase)
+            true
+          else
+            false
+          end
+        else
+          false
+        end
+      }
     else
       lambda { |ref| true }
     end
   end
     
-  def all_formulae
-    references = {}
-    worksheets do |name,xml_filename|
-      r = references[name] = {}
-      i = input([name,'Formulae'])
-      i.each_line do |line|
-        line =~ /^(.*?)\t(.*)$/
-        ref, ast = $1, $2
-        r[ref] = eval(ast)
-      end
-    end 
-    references
-  end
-  
   def c_name_for_worksheet_name(name)
-    unless @worksheet_names
-      w = input('Worksheet C names')
-      @worksheet_names = Hash[w.readlines.map { |line| line.split("\t").map { |a| a.strip }}]
-      close(w)
-    end
-    @worksheet_names[name]
+    @worksheet_c_names[name.to_s]
   end
     
-  def worksheets(&block)
-    unless @worksheet_filenames
-      worksheet_names = input('Worksheet names')
-      @worksheet_filenames = worksheet_names.each_line.map do |line|
-        name, filename = *line.split("\t")
-        [name, filename.strip]
-      end
-      close(worksheet_names)
+  def worksheets
+    @worksheet_xmls.each do |name, filename|
+      yield name, filename
     end
-    
-    @worksheet_filenames.each do |name, filename|
-      block.call(name, filename)
-    end
-  end
-    
-  def extract(klass,xml_name,output_name)
-    log.debug "Started using #{klass} to extract xml: #{xml_name} to #{output_name}"
-    
-    i = xml(xml_name)
-    o = intermediate(output_name)
-    klass.extract(i,o)
-    close(i,o)
-
-    log.info "Finished using #{klass} to extract xml: #{xml_name} to #{output_name}"
-  end
-  
-  def apply_rewrite(klass,filename)
-    rewrite klass, filename, filename
   end
   
-  def rewrite(klass, *args)
-    execute klass, :rewrite, *args
-  end
-  
-  def replace(klass, *args)
-    execute klass, :replace, *args
-  end
-  
-  def execute(klass, method, *args)
-    log.debug "Started executing #{klass}.#{method} with #{args.inspect}"
-    inputs = args[0..-2].map { |name| input(name) }
-    output = intermediate(args.last)
-    klass.send(method,*inputs,output)
-    close(*inputs,output)
-    log.info "Finished executing #{klass}.#{method} with #{args.inspect}"
-  end
-  
-  def xml(*args)
+  def xml(*args, &block)
     args.flatten!
     filename = File.join(xml_directory,'xl',*args)
     if File.exists?(filename)
-      File.open(filename,'r')
+      f = File.open(filename,'r')
     else
       log.warn("#{filename} does not exist in xml(#{args.inspect}), using blank instead")
-      StringIO.new
-    end
-  end
-  
-  def input(*args)
-    args.flatten!
-    filename = versioned_filename_read(intermediate_directory,*args)
-    if run_in_memory
-      existing_file = @files[filename]
-      if existing_file
-        StringIO.new(existing_file.string,'r')
-      else
-        log.warn("#{filename} does not exist in input(#{args.inspect}), using blank instead")
-        StringIO.new
-      end
-    else
-      if File.exists?(filename)
-        File.open(filename,'r')
-      else
-        log.warn("#{filename} does not exist in input(#{args.inspect}), using blank instead")
-        StringIO.new
-      end
+      f = StringIO.new
     end
-  end
-  
-  def intermediate(*args)
-    args.flatten!
-    filename = versioned_filename_write(intermediate_directory,*args)
-    if run_in_memory
-      @files ||= {}
-      remove_obsolete_versioned_filenames(intermediate_directory, *args)
-      @files[filename] = StringIO.new("",'w')
+    if block
+      yield f
+      f.close if f.respond_to?(:close)
     else
-      FileUtils.mkdir_p(File.dirname(filename))
-      File.open(filename,'w')
+      f
     end
   end
   
@@ -1037,43 +1055,4 @@ class ExcelToX
     @ruby_module_name
   end
 
-  def remove_obsolete_versioned_filenames(*args)
-    return unless run_in_memory
-    standardised_name = standardise_name(args)
-    counter = @versioned_filenames[standardised_name] || 0
-    0.upto(counter-1).map do |c|
-      @files.delete(filename_with_counter(c, args))
-    end
-  end
-  
-  def versioned_filename_read(*args)
-    @versioned_filenames ||= {}
-    standardised_name = standardise_name(args)
-    counter = @versioned_filenames[standardised_name]
-    filename_with_counter counter, args
-  end
-  
-  def versioned_filename_write(*args)
-    @versioned_filenames ||= {}
-    standardised_name = standardise_name(args)
-    if @versioned_filenames.has_key?(standardised_name)
-      counter =  @versioned_filenames[standardised_name] + 1
-    else
-      counter = 0
-    end
-    @versioned_filenames[standardised_name] = counter
-    filename_with_counter(counter, args)
-  end
-  
-  def filename_with_counter(counter, args)
-    counter ||= 0
-    last_name = args.last
-    last_name = last_name + sprintf(" %03d", counter)
-    File.join(*args[0..-2], last_name)    
-  end  
-  
-  def standardise_name(*args)
-    File.expand_path(File.join(args))
-  end
-  
 end