excel_to_code 0.0.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (123) hide show
  1. data/README +41 -0
  2. data/bin/excel_to_c +63 -0
  3. data/bin/excel_to_ruby +9 -0
  4. data/src/commands.rb +2 -0
  5. data/src/commands/excel_to_c.rb +858 -0
  6. data/src/commands/excel_to_ruby.rb +620 -0
  7. data/src/compile.rb +2 -0
  8. data/src/compile/c.rb +5 -0
  9. data/src/compile/c/compile_to_c.rb +62 -0
  10. data/src/compile/c/compile_to_c_header.rb +26 -0
  11. data/src/compile/c/compile_to_c_unit_test.rb +42 -0
  12. data/src/compile/c/excel_to_c_runtime.c +2029 -0
  13. data/src/compile/c/map_formulae_to_c.rb +184 -0
  14. data/src/compile/c/map_sheet_names_to_c_names.rb +19 -0
  15. data/src/compile/c/map_values_to_c.rb +85 -0
  16. data/src/compile/c/map_values_to_c_structs.rb +37 -0
  17. data/src/compile/ruby.rb +3 -0
  18. data/src/compile/ruby/compile_to_ruby.rb +33 -0
  19. data/src/compile/ruby/compile_to_ruby_unit_test.rb +28 -0
  20. data/src/compile/ruby/excel_to_ruby_runtime.rb +1 -0
  21. data/src/compile/ruby/map_formulae_to_ruby.rb +95 -0
  22. data/src/compile/ruby/map_sheet_names_to_ruby_names.rb +19 -0
  23. data/src/compile/ruby/map_values_to_ruby.rb +65 -0
  24. data/src/excel.rb +5 -0
  25. data/src/excel/area.rb +93 -0
  26. data/src/excel/excel_functions.rb +84 -0
  27. data/src/excel/excel_functions/abs.rb +14 -0
  28. data/src/excel/excel_functions/add.rb +18 -0
  29. data/src/excel/excel_functions/and.rb +30 -0
  30. data/src/excel/excel_functions/apply_to_range.rb +17 -0
  31. data/src/excel/excel_functions/average.rb +12 -0
  32. data/src/excel/excel_functions/choose.rb +18 -0
  33. data/src/excel/excel_functions/cosh.rb +9 -0
  34. data/src/excel/excel_functions/count.rb +9 -0
  35. data/src/excel/excel_functions/counta.rb +8 -0
  36. data/src/excel/excel_functions/divide.rb +23 -0
  37. data/src/excel/excel_functions/excel_equal.rb +20 -0
  38. data/src/excel/excel_functions/excel_if.rb +8 -0
  39. data/src/excel/excel_functions/excel_match.rb +51 -0
  40. data/src/excel/excel_functions/find.rb +39 -0
  41. data/src/excel/excel_functions/iferror.rb +10 -0
  42. data/src/excel/excel_functions/index.rb +48 -0
  43. data/src/excel/excel_functions/left.rb +12 -0
  44. data/src/excel/excel_functions/less_than.rb +26 -0
  45. data/src/excel/excel_functions/less_than_or_equal.rb +26 -0
  46. data/src/excel/excel_functions/max.rb +12 -0
  47. data/src/excel/excel_functions/min.rb +12 -0
  48. data/src/excel/excel_functions/mod.rb +15 -0
  49. data/src/excel/excel_functions/more_than.rb +26 -0
  50. data/src/excel/excel_functions/more_than_or_equal.rb +26 -0
  51. data/src/excel/excel_functions/multiply.rb +24 -0
  52. data/src/excel/excel_functions/negative.rb +12 -0
  53. data/src/excel/excel_functions/not_equal.rb +19 -0
  54. data/src/excel/excel_functions/number_argument.rb +30 -0
  55. data/src/excel/excel_functions/pi.rb +7 -0
  56. data/src/excel/excel_functions/pmt.rb +16 -0
  57. data/src/excel/excel_functions/power.rb +18 -0
  58. data/src/excel/excel_functions/round.rb +13 -0
  59. data/src/excel/excel_functions/rounddown.rb +14 -0
  60. data/src/excel/excel_functions/roundup.rb +17 -0
  61. data/src/excel/excel_functions/string_join.rb +19 -0
  62. data/src/excel/excel_functions/subtotal.rb +13 -0
  63. data/src/excel/excel_functions/subtract.rb +18 -0
  64. data/src/excel/excel_functions/sum.rb +8 -0
  65. data/src/excel/excel_functions/sumif.rb +7 -0
  66. data/src/excel/excel_functions/sumifs.rb +74 -0
  67. data/src/excel/excel_functions/sumproduct.rb +32 -0
  68. data/src/excel/excel_functions/vlookup.rb +49 -0
  69. data/src/excel/formula_peg.rb +238 -0
  70. data/src/excel/formula_peg.txt +45 -0
  71. data/src/excel/reference.rb +56 -0
  72. data/src/excel/table.rb +108 -0
  73. data/src/excel_to_code.rb +7 -0
  74. data/src/extract.rb +13 -0
  75. data/src/extract/check_for_unknown_functions.rb +20 -0
  76. data/src/extract/extract_array_formulae.rb +23 -0
  77. data/src/extract/extract_formulae.rb +36 -0
  78. data/src/extract/extract_named_references.rb +38 -0
  79. data/src/extract/extract_relationships.rb +10 -0
  80. data/src/extract/extract_shared_formulae.rb +23 -0
  81. data/src/extract/extract_shared_strings.rb +20 -0
  82. data/src/extract/extract_simple_formulae.rb +18 -0
  83. data/src/extract/extract_table.rb +24 -0
  84. data/src/extract/extract_values.rb +29 -0
  85. data/src/extract/extract_worksheet_dimensions.rb +11 -0
  86. data/src/extract/extract_worksheet_names.rb +10 -0
  87. data/src/extract/extract_worksheet_table_relationships.rb +10 -0
  88. data/src/extract/simple_extract_from_xml.rb +19 -0
  89. data/src/rewrite.rb +10 -0
  90. data/src/rewrite/ast_copy_formula.rb +42 -0
  91. data/src/rewrite/ast_expand_array_formulae.rb +180 -0
  92. data/src/rewrite/rewrite_array_formulae.rb +71 -0
  93. data/src/rewrite/rewrite_array_formulae_to_arrays.rb +18 -0
  94. data/src/rewrite/rewrite_cell_references_to_include_sheet.rb +56 -0
  95. data/src/rewrite/rewrite_formulae_to_ast.rb +24 -0
  96. data/src/rewrite/rewrite_merge_formulae_and_values.rb +18 -0
  97. data/src/rewrite/rewrite_relationship_id_to_filename.rb +22 -0
  98. data/src/rewrite/rewrite_shared_formulae.rb +38 -0
  99. data/src/rewrite/rewrite_values_to_ast.rb +28 -0
  100. data/src/rewrite/rewrite_whole_row_column_references_to_areas.rb +90 -0
  101. data/src/rewrite/rewrite_worksheet_names.rb +20 -0
  102. data/src/simplify.rb +16 -0
  103. data/src/simplify/count_formula_references.rb +58 -0
  104. data/src/simplify/identify_dependencies.rb +56 -0
  105. data/src/simplify/identify_repeated_formula_elements.rb +37 -0
  106. data/src/simplify/inline_formulae.rb +77 -0
  107. data/src/simplify/map_formulae_to_values.rb +157 -0
  108. data/src/simplify/remove_cells.rb +18 -0
  109. data/src/simplify/replace_arrays_with_single_cells.rb +27 -0
  110. data/src/simplify/replace_blanks.rb +58 -0
  111. data/src/simplify/replace_common_elements_in_formulae.rb +19 -0
  112. data/src/simplify/replace_formulae_with_calculated_values.rb +21 -0
  113. data/src/simplify/replace_indirects_with_references.rb +44 -0
  114. data/src/simplify/replace_named_references.rb +82 -0
  115. data/src/simplify/replace_ranges_with_array_literals.rb +54 -0
  116. data/src/simplify/replace_shared_strings.rb +49 -0
  117. data/src/simplify/replace_table_references.rb +71 -0
  118. data/src/simplify/replace_values_with_constants.rb +47 -0
  119. data/src/simplify/simplify_arithmetic.rb +54 -0
  120. data/src/util.rb +2 -0
  121. data/src/util/not_supported_exception.rb +2 -0
  122. data/src/util/try.rb +9 -0
  123. metadata +207 -0
@@ -0,0 +1,620 @@
1
+ # coding: utf-8
2
+
3
+ require 'fileutils'
4
+ require_relative '../util'
5
+ require_relative '../excel'
6
+ require_relative '../extract'
7
+ require_relative '../rewrite'
8
+ require_relative '../simplify'
9
+ require_relative '../compile'
10
+
11
+ class ExcelToRuby
12
+
13
+ attr_accessor :excel_file, :output_directory, :xml_dir, :compiled_module_name, :values_that_can_be_set_at_runtime, :outputs_to_keep
14
+
15
+ def initialize
16
+ @values_that_can_be_set_at_runtime ||= {}
17
+ end
18
+
19
+ def go!
20
+ self.excel_file = File.expand_path(excel_file)
21
+ self.output_directory = File.expand_path(output_directory)
22
+ self.xml_dir = File.join(output_directory,'xml')
23
+
24
+ sort_out_output_directories
25
+ unzip_excel
26
+ process_workbook
27
+ extract_worksheets
28
+ Process.waitall
29
+ merge_table_files
30
+ rewrite_worksheets
31
+ Process.waitall
32
+ simplify_worksheets
33
+ Process.waitall
34
+ optimise_and_replace_indirect_loop
35
+ Process.waitall
36
+ replace_blanks
37
+ Process.waitall
38
+ remove_any_cells_not_needed_for_outputs
39
+ Process.waitall
40
+ inline_formulae_that_are_only_used_once
41
+ Process.waitall
42
+ separate_formulae_elements
43
+ Process.waitall
44
+ compile_workbook
45
+ compile_worksheets
46
+ Process.waitall
47
+ end
48
+
49
+ def sort_out_output_directories
50
+ FileUtils.mkdir_p(File.join(output_directory,'intermediate'))
51
+ FileUtils.mkdir_p(File.join(output_directory,'ruby','worksheets'))
52
+ FileUtils.mkdir_p(File.join(output_directory,'ruby','tests'))
53
+ end
54
+
55
+ def unzip_excel
56
+ puts `unzip -uo '#{excel_file}' -d '#{xml_dir}'`
57
+ end
58
+
59
+ def process_workbook
60
+ extract ExtractSharedStrings, 'sharedStrings.xml', 'shared_strings'
61
+
62
+ extract ExtractNamedReferences, 'workbook.xml', 'named_references'
63
+ rewrite RewriteFormulaeToAst, 'named_references', 'named_references.ast'
64
+
65
+ extract ExtractWorksheetNames, 'workbook.xml', 'worksheet_names_without_filenames'
66
+ extract ExtractRelationships, File.join('_rels','workbook.xml.rels'), 'workbook_relationships'
67
+ rewrite RewriteWorksheetNames, 'worksheet_names_without_filenames', 'workbook_relationships', 'worksheet_names'
68
+ rewrite MapSheetNamesToRubyNames, 'worksheet_names', 'worksheet_ruby_names'
69
+
70
+ extract_dimensions_from_worksheets
71
+ end
72
+
73
+ # Extracts each worksheets values and formulas
74
+ def extract_worksheets
75
+ worksheets("Initial data extract") do |name,xml_filename|
76
+ #fork do
77
+ $0 = "ruby initial extract #{name}"
78
+ initial_extract_from_worksheet(name,xml_filename)
79
+ #end
80
+ end
81
+ end
82
+
83
+ # Extracts the dimensions of each worksheet and puts them in a single file
84
+ def extract_dimensions_from_worksheets
85
+ dimension_file = output('dimensions')
86
+ worksheets("Extracting dimensions") do |name,xml_filename|
87
+ dimension_file.write name
88
+ dimension_file.write "\t"
89
+ extract ExtractWorksheetDimensions, File.open(xml_filename,'r'), dimension_file
90
+ end
91
+ dimension_file.close
92
+ end
93
+
94
+ def rewrite_worksheets
95
+ worksheets("Initial rewrite of references and formulae") do |name,xml_filename|
96
+ #fork do
97
+ rewrite_row_and_column_references(name,xml_filename)
98
+ rewrite_shared_formulae(name,xml_filename)
99
+ rewrite_array_formulae(name,xml_filename)
100
+ combine_formulae_files(name,xml_filename)
101
+ #end
102
+ end
103
+ end
104
+
105
+ def rewrite_row_and_column_references(name,xml_filename)
106
+ dimensions = input('dimensions')
107
+ %w{simple_formulae.ast shared_formulae.ast array_formulae.ast}.each do |file|
108
+ dimensions.rewind
109
+ i = File.open(File.join(output_directory,'intermediate',name,file),'r')
110
+ o = File.open(File.join(output_directory,'intermediate',name,"#{file}-nocols"),'w')
111
+ RewriteWholeRowColumnReferencesToAreas.rewrite(i,name, dimensions, o)
112
+ close(i,o)
113
+ end
114
+ dimensions.close
115
+ end
116
+
117
+ def rewrite_shared_formulae(name,xml_filename)
118
+ i = File.open(File.join(output_directory,'intermediate',name,'shared_formulae.ast-nocols'),'r')
119
+ o = File.open(File.join(output_directory,'intermediate',name,"shared_formulae-expanded.ast"),'w')
120
+ RewriteSharedFormulae.rewrite(i,o)
121
+ close(i,o)
122
+ end
123
+
124
+ def rewrite_array_formulae(name,xml_filename)
125
+ r = ReplaceNamedReferences.new
126
+ r.sheet_name = name
127
+ replace r, File.join(name,'array_formulae.ast-nocols'), 'named_references.ast', File.join(name,"array_formulae1.ast")
128
+
129
+ r = ReplaceTableReferences.new
130
+ r.sheet_name = name
131
+ replace r, File.join(name,'array_formulae1.ast'), 'all_tables', File.join(name,"array_formulae2.ast")
132
+ replace SimplifyArithmetic, File.join(name,'array_formulae2.ast'), File.join(name,'array_formulae3.ast')
133
+ replace ReplaceRangesWithArrayLiterals, File.join(name,"array_formulae3.ast"), File.join(name,"array_formulae4.ast")
134
+ rewrite RewriteArrayFormulaeToArrays, File.join(name,"array_formulae4.ast"), File.join(name,"array_formulae5.ast")
135
+ rewrite RewriteArrayFormulae, File.join(name,'array_formulae5.ast'), File.join(name,"array_formulae-expanded.ast")
136
+ end
137
+
138
+ def combine_formulae_files(name,xml_filename)
139
+ values = File.join(name,'values.ast')
140
+ shared_formulae = File.join(name,"shared_formulae-expanded.ast")
141
+ array_formulae = File.join(name,"array_formulae-expanded.ast")
142
+ simple_formulae = File.join(name,"simple_formulae.ast-nocols")
143
+ output = File.join(name,'formulae.ast')
144
+ rewrite RewriteMergeFormulaeAndValues, values, shared_formulae, array_formulae, simple_formulae, output
145
+ end
146
+
147
+ def initial_extract_from_worksheet(name,xml_filename)
148
+ worksheet_directory = File.join(output_directory,'intermediate',name)
149
+ FileUtils.mkdir_p(worksheet_directory)
150
+ worksheet_xml = File.open(xml_filename,'r')
151
+ { ExtractValues => 'values',
152
+ ExtractSimpleFormulae => 'simple_formulae',
153
+ ExtractSharedFormulae => 'shared_formulae',
154
+ ExtractArrayFormulae => 'array_formulae'
155
+ }.each do |_klass,output_filename|
156
+ worksheet_xml.rewind
157
+ extract _klass, worksheet_xml, File.join(name,output_filename)
158
+ if _klass == ExtractValues
159
+ rewrite RewriteValuesToAst, File.join(name,output_filename), File.join(name,"#{output_filename}.ast")
160
+ else
161
+ rewrite RewriteFormulaeToAst, File.join(name,output_filename), File.join(name,"#{output_filename}.ast")
162
+ end
163
+ end
164
+ worksheet_xml.rewind
165
+ extract ExtractWorksheetTableRelationships, worksheet_xml, File.join(name,'table_rids')
166
+ if File.exists?(File.join(xml_dir,'xl','worksheets','_rels',"#{File.basename(xml_filename)}.rels"))
167
+ extract ExtractRelationships, File.join('worksheets','_rels',"#{File.basename(xml_filename)}.rels"), File.join(name,'relationships')
168
+ rewrite RewriteRelationshipIdToFilename, File.join(name,'table_rids'), File.join(name,'relationships'), File.join(name,'table_filenames')
169
+ tables = output(name,'tables')
170
+ table_extractor = ExtractTable.new(name)
171
+ table_filenames = input(name,'table_filenames')
172
+ table_filenames.lines.each do |line|
173
+ extract table_extractor, File.join('worksheets',line.strip), tables
174
+ end
175
+ close(tables,table_filenames)
176
+ else
177
+ FileUtils.touch File.join(output_directory,'intermediate',name,'relationships')
178
+ FileUtils.touch File.join(output_directory,'intermediate',name,'table_filenames')
179
+ FileUtils.touch File.join(output_directory,'intermediate',name,'tables')
180
+ end
181
+ close(worksheet_xml)
182
+ end
183
+
184
+ def merge_table_files
185
+ tables = []
186
+ worksheets("Merging table files") do |name,xml_filename|
187
+ tables << File.join(output_directory,'intermediate',name,'tables')
188
+ end
189
+ `sort #{tables.map { |t| " '#{t}' "}.join} > #{File.join(output_directory,'intermediate','all_tables')}`
190
+ end
191
+
192
+ def simplify_worksheets
193
+ worksheets("Simplifying") do |name,xml_filename|
194
+ #fork do
195
+ # i = input( File.join(name,'formulae.ast'))
196
+ # o = output(File.join(name,'missing_functions'))
197
+ # CheckForUnknownFunctions.new.check(i,o)
198
+ # close(i,o)
199
+ simplify_worksheet(name,xml_filename)
200
+ #end
201
+ end
202
+ # missing_function_files = []
203
+ # worksheets("Consolidating any missing functions") do |name,xml_filename|
204
+ # missing_function_files << File.join(output_directory,'intermediate',name,'missing_functions')
205
+ # end
206
+ # `sort -u #{missing_function_files.map { |t| " '#{t}' "}.join} > #{File.join(output_directory,'intermediate','all_missing_functions')}`
207
+ end
208
+
209
+ def simplify_worksheet(name,xml_filename)
210
+ replace SimplifyArithmetic, File.join(name,'formulae.ast'), File.join(name,'formulae_simple_arithmetic.ast')
211
+ replace ReplaceSharedStrings, File.join(name,'formulae_simple_arithmetic.ast'), 'shared_strings', File.join(name,"formulae_no_shared_strings.ast")
212
+ replace ReplaceSharedStrings, File.join(name,'values.ast'), 'shared_strings', File.join(name,"values_no_shared_strings.ast")
213
+ r = ReplaceNamedReferences.new
214
+ r.sheet_name = name
215
+ replace r, File.join(name,'formulae_no_shared_strings.ast'), 'named_references.ast', File.join(name,"formulae_no_named_references.ast")
216
+
217
+ r = ReplaceTableReferences.new
218
+ r.sheet_name = name
219
+ replace r, File.join(name,'formulae_no_named_references.ast'), 'all_tables', File.join(name,"formulae_no_table_references.ast")
220
+ replace ReplaceRangesWithArrayLiterals, File.join(name,"formulae_no_table_references.ast"), File.join(name,"formulae_no_ranges.ast")
221
+ end
222
+
223
+ def replace_blanks
224
+ references = {}
225
+ worksheets("Loading formulae") do |name,xml_filename|
226
+ r = references[name] = {}
227
+ i = input(name,"formulae_no_indirects_optimised.ast")
228
+ i.lines do |line|
229
+ ref = line[/^(.*?)\t/,1]
230
+ r[ref] = true
231
+ end
232
+ end
233
+ worksheets("Replacing blanks") do |name,xml_filename|
234
+ #fork do
235
+ r = ReplaceBlanks.new
236
+ r.references = references
237
+ r.default_sheet_name = name
238
+ replace r, File.join(name,"formulae_no_indirects_optimised.ast"),File.join(name,"formulae_no_blanks.ast")
239
+ #end
240
+ end
241
+ end
242
+
243
+ def optimise_and_replace_indirect_loop
244
+ number_of_loops = 4
245
+ 1.upto(number_of_loops) do |pass|
246
+ puts "Optimise and replace indirects pass #{pass}"
247
+ start = pass == 1 ? "formulae_no_ranges.ast" : "optimse-output-#{pass-1}.ast"
248
+ finish = pass == number_of_loops ? "formulae_no_indirects_optimised.ast" : "optimse-output-#{pass}.ast"
249
+ replace_indirects(start,"replace-indirect-output-#{pass}.ast","replace-indirect-working-#{pass}-")
250
+ optimise_sheets("replace-indirect-output-#{pass}.ast",finish,"optimse-working-#{pass}-")
251
+ end
252
+ end
253
+
254
+ def replace_indirects(start_filename,finish_filename,basename)
255
+ worksheets("Replacing indirects") do |name,xml_filename|
256
+ counter = 1
257
+ replace ReplaceIndirectsWithReferences, File.join(name,start_filename), File.join(name,"#{basename}#{counter+1}.ast")
258
+ counter += 1
259
+
260
+ r = ReplaceNamedReferences.new
261
+ r.sheet_name = name
262
+ replace r, File.join(name,"#{basename}#{counter}.ast"), 'named_references.ast', File.join(name,"#{basename}#{counter+1}.ast")
263
+ counter += 1
264
+
265
+ r = ReplaceTableReferences.new
266
+ r.sheet_name = name
267
+ replace r, File.join(name,"#{basename}#{counter}.ast"), 'all_tables', File.join(name,"#{basename}#{counter+1}.ast")
268
+ counter += 1
269
+
270
+ replace ReplaceRangesWithArrayLiterals, File.join(name,"#{basename}#{counter}.ast"), File.join(name,"#{basename}#{counter+1}.ast")
271
+ counter += 1
272
+
273
+ # Finally, create the output directory
274
+ i = File.join(output_directory,'intermediate',name,"#{basename}#{counter}.ast")
275
+ o = File.join(output_directory,'intermediate',name,finish_filename)
276
+ `cp '#{i}' '#{o}'`
277
+ end
278
+ end
279
+
280
+ def optimise_sheets(start_filename,finish_filename,basename)
281
+ counter = 1
282
+
283
+ # Setup start
284
+ worksheets("Setting up for optimise") do |name|
285
+ i = File.join(output_directory,'intermediate',name,start_filename)
286
+ o = File.join(output_directory,'intermediate',name,"#{basename}#{counter}.ast")
287
+ `cp '#{i}' '#{o}'`
288
+ end
289
+
290
+ worksheets("Replacing with calculated values") do |name,xml_filename|
291
+ #fork do
292
+ replace ReplaceFormulaeWithCalculatedValues, File.join(name,"#{basename}#{counter}.ast"), File.join(name,"#{basename}#{counter+1}.ast")
293
+ #end
294
+ end
295
+ counter += 1
296
+ Process.waitall
297
+
298
+ references = all_formulae("#{basename}#{counter}.ast")
299
+ inline_ast_decision = lambda do |sheet,cell,references|
300
+ references_to_keep = @values_that_can_be_set_at_runtime[sheet]
301
+ if references_to_keep && (references_to_keep == :all || references_to_keep.include?(cell))
302
+ false
303
+ else
304
+ ast = references[sheet][cell]
305
+ if ast
306
+ if [:number,:string,:blank,:null,:error,:boolean_true,:boolean_false,:sheet_reference,:cell].include?(ast.first)
307
+ # puts "Inlining #{sheet}.#{cell}: #{ast.inspect}"
308
+ true
309
+ else
310
+ false
311
+ end
312
+ else
313
+ true # Always inline blanks
314
+ end
315
+ end
316
+ end
317
+ r = InlineFormulae.new
318
+ r.references = references
319
+ r.inline_ast = inline_ast_decision
320
+
321
+ worksheets("Inlining formulae") do |name,xml_filename|
322
+ #fork do
323
+ r.default_sheet_name = name
324
+ replace r, File.join(name,"#{basename}#{counter}.ast"), File.join(name,"#{basename}#{counter+1}.ast")
325
+ #end
326
+ end
327
+ counter += 1
328
+ Process.waitall
329
+
330
+ # Finish
331
+ worksheets("Moving sheets") do |name|
332
+ o = File.join(output_directory,'intermediate',name,finish_filename)
333
+ i = File.join(output_directory,'intermediate',name,"#{basename}#{counter}.ast")
334
+ `cp '#{i}' '#{o}'`
335
+ end
336
+ end
337
+
338
+ def remove_any_cells_not_needed_for_outputs(formula_in = "formulae_no_blanks.ast", formula_out = "formulae_pruned.ast", values_in = "values_no_shared_strings.ast", values_out = "values_pruned.ast")
339
+ if outputs_to_keep && !outputs_to_keep.empty?
340
+ identifier = IdentifyDependencies.new
341
+ identifier.references = all_formulae(formula_in)
342
+ outputs_to_keep.each do |sheet_to_keep,cells_to_keep|
343
+ if cells_to_keep == :all
344
+ identifier.add_depedencies_for(sheet_to_keep)
345
+ elsif cells_to_keep.is_a?(Array)
346
+ cells_to_keep.each do |cell|
347
+ identifier.add_depedencies_for(sheet_to_keep,cell)
348
+ end
349
+ end
350
+ end
351
+ r = RemoveCells.new
352
+ worksheets("Removing cells") do |name,xml_filename|
353
+ #fork do
354
+ r.cells_to_keep = identifier.dependencies[name]
355
+ rewrite r, File.join(name, formula_in), File.join(name, formula_out)
356
+ rewrite r, File.join(name, values_in), File.join(name, values_out)
357
+ #end
358
+ end
359
+ Process.waitall
360
+ else
361
+ worksheets do |name,xml_filename|
362
+ i = File.join(output_directory,'intermediate',name, formula_in)
363
+ o = File.join(output_directory,'intermediate',name, formula_out)
364
+ `cp '#{i}' '#{o}'`
365
+ i = File.join(output_directory,'intermediate',name, values_in)
366
+ o = File.join(output_directory,'intermediate',name, values_out)
367
+ `cp '#{i}' '#{o}'`
368
+ end
369
+ end
370
+ end
371
+
372
+ def inline_formulae_that_are_only_used_once
373
+ references = all_formulae("formulae_pruned.ast")
374
+ counter = CountFormulaReferences.new
375
+ count = counter.count(references)
376
+
377
+ inline_ast_decision = lambda do |sheet,cell,references|
378
+ references_to_keep = @values_that_can_be_set_at_runtime[sheet]
379
+ if references_to_keep && (references_to_keep == :all || references_to_keep.include?(cell))
380
+ false
381
+ else
382
+ count[sheet][cell] == 1
383
+ end
384
+ end
385
+
386
+ r = InlineFormulae.new
387
+ r.references = references
388
+ r.inline_ast = inline_ast_decision
389
+
390
+ worksheets("Inlining formulae") do |name,xml_filename|
391
+ #fork do
392
+ r.default_sheet_name = name
393
+ replace r, File.join(name,"formulae_pruned.ast"), File.join(name,"formulae_inlined.ast")
394
+ #end
395
+ end
396
+
397
+ remove_any_cells_not_needed_for_outputs("formulae_inlined.ast", "formulae_inlined_pruned.ast", "values_pruned.ast", "values_pruned2.ast")
398
+
399
+ # worksheets("Skipping inlining") do |name,xml_filename|
400
+ # i = File.join(output_directory,'intermediate',name, "formulae_pruned.ast")
401
+ # o = File.join(output_directory,'intermediate',name, "formulae_inlined_pruned.ast")
402
+ # `cp '#{i}' '#{o}'`
403
+ # i = File.join(output_directory,'intermediate',name, "values_pruned.ast")
404
+ # o = File.join(output_directory,'intermediate',name, "values_pruned2.ast")
405
+ # `cp '#{i}' '#{o}'`
406
+ # end
407
+
408
+ end
409
+
410
+ def separate_formulae_elements
411
+ # First we add the sheet to all references, so that we can then look for common elements accross worksheets
412
+ r = RewriteCellReferencesToIncludeSheet.new
413
+ worksheets("Adding the sheet to all references") do |name,xml_filename|
414
+ r.worksheet = name
415
+ rewrite r, File.join(name,"formulae_inlined_pruned.ast"), File.join(name,"formulae_inlined_pruned_with_sheets.ast")
416
+ end
417
+
418
+ references = all_formulae("formulae_inlined_pruned_with_sheets.ast")
419
+ identifier = IdentifyRepeatedFormulaElements.new
420
+ repeated_elements = identifier.count(references)
421
+ repeated_elements.delete_if do |element,count|
422
+ count < 2
423
+ end
424
+ o = output('common-elements.ast')
425
+ i = 0
426
+ repeated_elements.each do |element,count|
427
+ o.puts "common#{i}\t#{element}"
428
+ i = i + 1
429
+ end
430
+ close(o)
431
+
432
+ worksheets("Replacing repeated elements") do |name,xml_filename|
433
+ replace ReplaceCommonElementsInFormulae, File.join(name,"formulae_inlined_pruned_with_sheets.ast"), "common-elements.ast", File.join(name,"formulae_inlined_pruned_replaced.ast")
434
+ end
435
+ end
436
+
437
+ def all_formulae(filename)
438
+ references = {}
439
+ worksheets do |name,xml_filename|
440
+ r = references[name] = {}
441
+ i = input(name,filename)
442
+ i.lines do |line|
443
+ line =~ /^(.*?)\t(.*)$/
444
+ ref, ast = $1, $2
445
+ r[$1] = eval($2)
446
+ end
447
+ end
448
+ references
449
+ end
450
+
451
+ def compile_workbook
452
+ compile_workbook_code
453
+ compile_workbook_test
454
+ end
455
+
456
+ def compile_workbook_code
457
+ w = input("worksheet_ruby_names")
458
+ o = ruby("#{compiled_module_name.downcase}.rb")
459
+ o.puts "# Compiled version of #{excel_file}"
460
+ o.puts "require '#{File.expand_path(File.join(File.dirname(__FILE__),'../excel/excel_functions'))}'"
461
+ o.puts ""
462
+ o.puts "module #{compiled_module_name}"
463
+ o.puts "class Spreadsheet"
464
+ o.puts " include ExcelFunctions"
465
+ w.lines do |line|
466
+ name, ruby_name = line.strip.split("\t")
467
+ o.puts " def #{ruby_name}; @#{ruby_name} ||= #{ruby_name.capitalize}.new; end"
468
+ end
469
+
470
+ c = CompileToRuby.new
471
+ i = input("common-elements.ast")
472
+ w.rewind
473
+ c.rewrite(i,w,o)
474
+
475
+ o.puts "end"
476
+ o.puts 'Dir[File.join(File.dirname(__FILE__),"worksheets/","*.rb")].each {|f| autoload(File.basename(f,".rb").capitalize,f)}'
477
+ o.puts "end"
478
+ close(i,w,o)
479
+ end
480
+
481
+ def compile_workbook_test
482
+ w = input("worksheet_ruby_names")
483
+ o = ruby("test_#{compiled_module_name.downcase}.rb")
484
+ o.puts "# All tests for #{excel_file}"
485
+ o.puts "require 'test/unit'"
486
+ w.lines do |line|
487
+ name, ruby_name = line.strip.split("\t")
488
+ o.puts "require_relative 'tests/test_#{ruby_name.downcase}'"
489
+ end
490
+ close(w,o)
491
+ end
492
+
493
+ def compile_worksheets
494
+ worksheets("Compiling worksheet") do |name,xml_filename|
495
+ #fork do
496
+ compile_worksheet_code(name,xml_filename)
497
+ compile_worksheet_test(name,xml_filename)
498
+ #end
499
+ end
500
+ end
501
+
502
+ def compile_worksheet_code(name,xml_filename)
503
+ settable_refs = @values_that_can_be_set_at_runtime[name]
504
+ c = CompileToRuby.new
505
+ c.settable =lambda { |ref| (settable_refs == :all) ? true : settable_refs.include?(ref) } if settable_refs
506
+ c.worksheet = name
507
+ i = input(name,"formulae_inlined_pruned_replaced.ast")
508
+ w = input("worksheet_ruby_names")
509
+ ruby_name = ruby_name_for_worksheet_name(name)
510
+ o = ruby('worksheets',"#{ruby_name.downcase}.rb")
511
+ d = output(name,'defaults')
512
+ o.puts "# coding: utf-8"
513
+ o.puts "# #{name}"
514
+ o.puts
515
+ o.puts "require_relative '../#{compiled_module_name.downcase}'"
516
+ o.puts
517
+ o.puts "module #{compiled_module_name}"
518
+ o.puts "class #{ruby_name.capitalize} < Spreadsheet"
519
+ c.rewrite(i,w,o,d)
520
+ o.puts ""
521
+ close(d)
522
+ if settable_refs
523
+ o.puts " def initialize"
524
+ d = input(name,'defaults')
525
+ d.lines do |line|
526
+ o.puts line
527
+ end
528
+ o.puts " end"
529
+ o.puts ""
530
+ close(d)
531
+ end
532
+ o.puts "end"
533
+ o.puts "end"
534
+ close(i,o)
535
+ end
536
+
537
+ def compile_worksheet_test(name,xml_filename)
538
+ i = input(name,"values_pruned2.ast")
539
+ ruby_name = ruby_name_for_worksheet_name(name)
540
+ o = ruby('tests',"test_#{ruby_name.downcase}.rb")
541
+ o.puts "# coding: utf-8"
542
+ o.puts "# Test for #{name}"
543
+ o.puts "require 'test/unit'"
544
+ o.puts "require_relative '../#{compiled_module_name.downcase}'"
545
+ o.puts
546
+ o.puts "module #{compiled_module_name}"
547
+ o.puts "class Test#{ruby_name.capitalize} < Test::Unit::TestCase"
548
+ o.puts " def spreadsheet; $spreadsheet ||= Spreadsheet.new; end"
549
+ o.puts " def worksheet; @worksheet ||= spreadsheet.#{ruby_name}; end"
550
+ CompileToRubyUnitTest.rewrite(i, o)
551
+ o.puts "end"
552
+ o.puts "end"
553
+ close(i,o)
554
+ end
555
+
556
+ def ruby_name_for_worksheet_name(name)
557
+ unless @worksheet_names
558
+ w = input("worksheet_ruby_names")
559
+ @worksheet_names = Hash[w.readlines.map { |line| line.split("\t").map { |a| a.strip }}]
560
+ close(w)
561
+ end
562
+ @worksheet_names[name]
563
+ end
564
+
565
+ def worksheets(message = "Processing",&block)
566
+ IO.readlines(File.join(output_directory,'intermediate','worksheet_names')).each do |line|
567
+ name, filename = *line.split("\t")
568
+ filename = File.expand_path(File.join(xml_dir,'xl',filename.strip))
569
+ puts "#{message} #{name}"
570
+ block.call(name, filename)
571
+ end
572
+ end
573
+
574
+ def extract(_klass,xml_name,output_name)
575
+ i = xml_name.is_a?(String) ? xml(xml_name) : xml_name
576
+ o = output_name.is_a?(String) ? output(output_name) : output_name
577
+ _klass.extract(i,o)
578
+ if xml_name.is_a?(String)
579
+ close(i)
580
+ end
581
+ if output_name.is_a?(String)
582
+ close(o)
583
+ end
584
+ end
585
+
586
+ def rewrite(_klass,*args)
587
+ o = output(args.pop)
588
+ inputs = args.map { |name| input(name) }
589
+ _klass.rewrite(*inputs,o)
590
+ close(*inputs,o)
591
+ end
592
+
593
+ def replace(_klass,*args)
594
+ o = output(args.pop)
595
+ inputs = args.map { |name| input(name) }
596
+ _klass.replace(*inputs,o)
597
+ close(*inputs,o)
598
+ end
599
+
600
+ def xml(*args)
601
+ File.open(File.join(xml_dir,'xl',*args),'r')
602
+ end
603
+
604
+ def input(*args)
605
+ File.open(File.join(output_directory,'intermediate',*args),'r')
606
+ end
607
+
608
+ def output(*args)
609
+ File.open(File.join(output_directory,'intermediate',*args),'w')
610
+ end
611
+
612
+ def ruby(*args)
613
+ File.open(File.join(output_directory,'ruby',*args),'w')
614
+ end
615
+
616
+ def close(*args)
617
+ args.map(&:close)
618
+ end
619
+
620
+ end