excel_to_code 0.0.1 → 0.0.2
Sign up to get free protection for your applications and to get access to all the features.
- data/bin/excel_to_c +11 -7
- data/src/commands.rb +1 -0
- data/src/commands/excel_to_c.rb +7 -615
- data/src/commands/excel_to_ruby.rb +14 -514
- data/src/commands/excel_to_x.rb +660 -0
- metadata +7 -6
@@ -0,0 +1,660 @@
|
|
1
|
+
# coding: utf-8
|
2
|
+
require 'fileutils'
|
3
|
+
require_relative '../excel_to_code'
|
4
|
+
|
5
|
+
# Used to throw normally fatal errors
|
6
|
+
class ExcelToCodeException < Exception; end
|
7
|
+
|
8
|
+
class ExcelToX
|
9
|
+
|
10
|
+
# Required attribute. The source excel file. This must be .xlsx not .xls
|
11
|
+
attr_accessor :excel_file
|
12
|
+
|
13
|
+
# Optional attribute. The output directory.
|
14
|
+
# If not specified, will be '#{excel_file_name}/c'
|
15
|
+
attr_accessor :output_directory
|
16
|
+
|
17
|
+
# Optional attribute. The name of the resulting c file (and associated ruby ffi module). Defaults to excelspreadsheet
|
18
|
+
attr_accessor :output_name
|
19
|
+
|
20
|
+
# Optional attribute. The excel file will be translated to xml and stored here.
|
21
|
+
# If not specified, will be '#{excel_file_name}/xml'
|
22
|
+
attr_accessor :xml_directory
|
23
|
+
|
24
|
+
# Optional attribute. The intermediate workings will be stored here.
|
25
|
+
# If not specified, will be '#{excel_file_name}/intermediate'
|
26
|
+
attr_accessor :intermediate_directory
|
27
|
+
|
28
|
+
# Optional attribute. Specifies which cells have setters created in the c code so their values can be altered at runtime.
|
29
|
+
# It is a hash. The keys are the sheet names. The values are either the symbol :all to specify that all cells on that sheet
|
30
|
+
# should be setable, or an array of cell names on that sheet that should be settable (e.g., A1)
|
31
|
+
attr_accessor :cells_that_can_be_set_at_runtime
|
32
|
+
|
33
|
+
# Optional attribute. Specifies which cells must appear in the final generated code.
|
34
|
+
# The default is that all cells in the original spreadsheet appear in the final code.
|
35
|
+
#
|
36
|
+
# If specified, then any cells that are not:
|
37
|
+
# * specified
|
38
|
+
# * required to calculate one of the cells that is specified
|
39
|
+
# * specified as a cell that can be set at runtime
|
40
|
+
# may be excluded from the final generated code.
|
41
|
+
#
|
42
|
+
# It is a hash. The keys are the sheet names. The values are either the symbol :all to specify that all cells on that sheet
|
43
|
+
# should be lept, or an array of cell names on that sheet that should be kept (e.g., A1)
|
44
|
+
attr_accessor :cells_to_keep
|
45
|
+
|
46
|
+
# Optional attribute. Boolean. Not relevant to all types of code output
|
47
|
+
# * true - the generated c code is compiled
|
48
|
+
# * false - the generated c code is not compiled (default, unless actuall_run_tests is specified as true)
|
49
|
+
attr_accessor :actually_compile_code
|
50
|
+
|
51
|
+
# Optional attribute. Boolean.
|
52
|
+
# * true - the generated tests are run
|
53
|
+
# * false - the generated tests are not run
|
54
|
+
attr_accessor :actually_run_tests
|
55
|
+
|
56
|
+
# Optional attribute. Boolean.
|
57
|
+
# * true - the intermediate files are not written to disk (requires a lot of memory)
|
58
|
+
# * false - the intermediate files are written to disk (default, easier to debug)
|
59
|
+
attr_accessor :run_in_memory
|
60
|
+
|
61
|
+
def set_defaults
|
62
|
+
raise ExcelToCodeException.new("No excel file has been specified") unless excel_file
|
63
|
+
|
64
|
+
self.output_directory ||= File.join(File.dirname(excel_file),File.basename(excel_file,".*"),'c')
|
65
|
+
self.xml_directory ||= File.join(File.dirname(excel_file),File.basename(excel_file,".*"),'xml')
|
66
|
+
self.intermediate_directory ||= File.join(File.dirname(excel_file),File.basename(excel_file,".*"),'intermediate')
|
67
|
+
|
68
|
+
self.output_name ||= "Excelspreadsheet"
|
69
|
+
|
70
|
+
self.cells_that_can_be_set_at_runtime ||= {}
|
71
|
+
|
72
|
+
# Make sure that all the cell names are downcase and don't have any $ in them
|
73
|
+
cells_that_can_be_set_at_runtime.keys.each do |sheet|
|
74
|
+
next unless cells_that_can_be_set_at_runtime[sheet].is_a?(Array)
|
75
|
+
cells_that_can_be_set_at_runtime[sheet] = cells_that_can_be_set_at_runtime[sheet].map { |reference| reference.gsub('$','').downcase }
|
76
|
+
end
|
77
|
+
|
78
|
+
# Make sure that all the cell names are downcase and don't have any $ in them
|
79
|
+
if cells_to_keep
|
80
|
+
cells_to_keep.keys.each do |sheet|
|
81
|
+
next unless cells_to_keep[sheet].is_a?(Array)
|
82
|
+
cells_to_keep[sheet] = cells_to_keep[sheet].map { |reference| reference.gsub('$','').downcase }
|
83
|
+
end
|
84
|
+
end
|
85
|
+
|
86
|
+
# Make sure the relevant directories exist
|
87
|
+
self.excel_file = File.expand_path(excel_file)
|
88
|
+
self.output_directory = File.expand_path(output_directory)
|
89
|
+
end
|
90
|
+
|
91
|
+
def go!
|
92
|
+
# This sorts out the attributes
|
93
|
+
set_defaults
|
94
|
+
|
95
|
+
# These turn the excel into a more accesible format
|
96
|
+
sort_out_output_directories
|
97
|
+
unzip_excel
|
98
|
+
|
99
|
+
# These get all the information out of the excel and put
|
100
|
+
# into a useful format
|
101
|
+
extract_data_from_workbook
|
102
|
+
extract_data_from_worksheets
|
103
|
+
merge_table_files
|
104
|
+
|
105
|
+
rewrite_worksheets
|
106
|
+
|
107
|
+
# These perform a series of transformations to the information
|
108
|
+
# with the intent of removing any redundant calculations
|
109
|
+
# that are in the excel
|
110
|
+
simplify_worksheets
|
111
|
+
optimise_and_replace_indirect_loop
|
112
|
+
replace_blanks
|
113
|
+
remove_any_cells_not_needed_for_outputs
|
114
|
+
inline_formulae_that_are_only_used_once
|
115
|
+
separate_formulae_elements
|
116
|
+
|
117
|
+
# This actually creates the code (implemented in subclasses)
|
118
|
+
write_code
|
119
|
+
|
120
|
+
# These compile and run the code version of the excel
|
121
|
+
compile_code
|
122
|
+
run_tests
|
123
|
+
|
124
|
+
puts
|
125
|
+
puts "The generated code is available in #{File.join(output_directory)}"
|
126
|
+
end
|
127
|
+
|
128
|
+
def sort_out_output_directories
|
129
|
+
FileUtils.mkdir_p(output_directory)
|
130
|
+
FileUtils.mkdir_p(xml_directory)
|
131
|
+
FileUtils.mkdir_p(intermediate_directory)
|
132
|
+
end
|
133
|
+
|
134
|
+
def unzip_excel
|
135
|
+
puts `rm -fr '#{xml_directory}'`
|
136
|
+
puts `unzip -uo '#{excel_file}' -d '#{xml_directory}'`
|
137
|
+
end
|
138
|
+
|
139
|
+
def extract_data_from_workbook
|
140
|
+
extract_shared_strings
|
141
|
+
extract_named_references
|
142
|
+
extract_worksheet_names
|
143
|
+
extract_dimensions_from_worksheets
|
144
|
+
end
|
145
|
+
|
146
|
+
def extract_shared_strings
|
147
|
+
if File.exists?(File.join(xml_directory,'xl','sharedStrings.xml'))
|
148
|
+
extract ExtractSharedStrings, 'sharedStrings.xml', 'shared_strings'
|
149
|
+
else
|
150
|
+
FileUtils.touch(File.join(intermediate_directory,'shared_strings'))
|
151
|
+
end
|
152
|
+
end
|
153
|
+
|
154
|
+
def extract_named_references
|
155
|
+
extract ExtractNamedReferences, 'workbook.xml', 'named_references'
|
156
|
+
rewrite RewriteFormulaeToAst, 'named_references', 'named_references.ast'
|
157
|
+
end
|
158
|
+
|
159
|
+
def extract_worksheet_names
|
160
|
+
extract ExtractWorksheetNames, 'workbook.xml', 'worksheet_names_without_filenames'
|
161
|
+
extract ExtractRelationships, File.join('_rels','workbook.xml.rels'), 'workbook_relationships'
|
162
|
+
rewrite RewriteWorksheetNames, 'worksheet_names_without_filenames', 'workbook_relationships', 'worksheet_names'
|
163
|
+
rewrite MapSheetNamesToCNames, 'worksheet_names', 'worksheet_c_names'
|
164
|
+
end
|
165
|
+
|
166
|
+
def extract_dimensions_from_worksheets
|
167
|
+
dimension_file = intermediate('dimensions')
|
168
|
+
worksheets("Extracting dimensions") do |name,xml_filename|
|
169
|
+
dimension_file.write name
|
170
|
+
dimension_file.write "\t"
|
171
|
+
extract ExtractWorksheetDimensions, File.open(xml_filename,'r'), dimension_file
|
172
|
+
end
|
173
|
+
dimension_file.close
|
174
|
+
end
|
175
|
+
|
176
|
+
def extract_data_from_worksheets
|
177
|
+
worksheets("Initial data extract") do |name,xml_filename|
|
178
|
+
worksheet_directory = File.join(intermediate_directory,name)
|
179
|
+
worksheet_xml = File.open(xml_filename,'r')
|
180
|
+
|
181
|
+
worksheet_xml.rewind
|
182
|
+
extract ExtractValues, worksheet_xml, File.join(name,'values')
|
183
|
+
rewrite RewriteValuesToAst, File.join(name,'values'), File.join(name,'values.ast')
|
184
|
+
|
185
|
+
worksheet_xml.rewind
|
186
|
+
extract ExtractSimpleFormulae, worksheet_xml, File.join(name,'simple_formulae')
|
187
|
+
rewrite RewriteFormulaeToAst, File.join(name,'simple_formulae'), File.join(name,'simple_formulae.ast')
|
188
|
+
|
189
|
+
worksheet_xml.rewind
|
190
|
+
extract ExtractSharedFormulae, worksheet_xml, File.join(name,'shared_formulae')
|
191
|
+
rewrite RewriteFormulaeToAst, File.join(name,'shared_formulae'), File.join(name,'shared_formulae.ast')
|
192
|
+
|
193
|
+
worksheet_xml.rewind
|
194
|
+
extract ExtractArrayFormulae, worksheet_xml, File.join(name,'array_formulae')
|
195
|
+
rewrite RewriteFormulaeToAst, File.join(name,'array_formulae'), File.join(name,'array_formulae.ast')
|
196
|
+
|
197
|
+
worksheet_xml.rewind
|
198
|
+
extract ExtractWorksheetTableRelationships, worksheet_xml, File.join(name,'table_rids')
|
199
|
+
if File.exists?(File.join(xml_directory,'xl','worksheets','_rels',"#{File.basename(xml_filename)}.rels"))
|
200
|
+
extract_tables(name,xml_filename)
|
201
|
+
else
|
202
|
+
fake_extract_tables(name,xml_filename)
|
203
|
+
end
|
204
|
+
close(worksheet_xml)
|
205
|
+
end
|
206
|
+
end
|
207
|
+
|
208
|
+
def extract_tables(name,xml_filename)
|
209
|
+
extract ExtractRelationships, File.join('worksheets','_rels',"#{File.basename(xml_filename)}.rels"), File.join(name,'relationships')
|
210
|
+
rewrite RewriteRelationshipIdToFilename, File.join(name,'table_rids'), File.join(name,'relationships'), File.join(name,'table_filenames')
|
211
|
+
tables = intermediate(name,'tables')
|
212
|
+
table_extractor = ExtractTable.new(name)
|
213
|
+
table_filenames = input(name,'table_filenames')
|
214
|
+
table_filenames.lines.each do |line|
|
215
|
+
extract table_extractor, File.join('worksheets',line.strip), tables
|
216
|
+
end
|
217
|
+
close(tables,table_filenames)
|
218
|
+
end
|
219
|
+
|
220
|
+
def fake_extract_tables(name,xml_filename)
|
221
|
+
a = intermediate(name,'relationships')
|
222
|
+
b = intermediate(name,'table_filenames')
|
223
|
+
c = intermediate(name,'tables')
|
224
|
+
close(a,b,c)
|
225
|
+
end
|
226
|
+
|
227
|
+
def rewrite_worksheets
|
228
|
+
worksheets("Initial rewrite of references and formulae") do |name,xml_filename|
|
229
|
+
rewrite_row_and_column_references(name,xml_filename)
|
230
|
+
rewrite_shared_formulae(name,xml_filename)
|
231
|
+
rewrite_array_formulae(name,xml_filename)
|
232
|
+
combine_formulae_files(name,xml_filename)
|
233
|
+
end
|
234
|
+
end
|
235
|
+
|
236
|
+
def rewrite_row_and_column_references(name,xml_filename)
|
237
|
+
dimensions = input('dimensions')
|
238
|
+
%w{simple_formulae.ast shared_formulae.ast array_formulae.ast}.each do |file|
|
239
|
+
dimensions.rewind
|
240
|
+
i = input(name,file)
|
241
|
+
o = intermediate(name,"#{file}-nocols")
|
242
|
+
RewriteWholeRowColumnReferencesToAreas.rewrite(i,name, dimensions, o)
|
243
|
+
close(i,o)
|
244
|
+
end
|
245
|
+
dimensions.close
|
246
|
+
end
|
247
|
+
|
248
|
+
def rewrite_shared_formulae(name,xml_filename)
|
249
|
+
i = input(name,'shared_formulae.ast-nocols')
|
250
|
+
o = intermediate(name,"shared_formulae-expanded.ast")
|
251
|
+
RewriteSharedFormulae.rewrite(i,o)
|
252
|
+
close(i,o)
|
253
|
+
end
|
254
|
+
|
255
|
+
def rewrite_array_formulae(name,xml_filename)
|
256
|
+
r = ReplaceNamedReferences.new
|
257
|
+
r.sheet_name = name
|
258
|
+
replace r, File.join(name,'array_formulae.ast-nocols'), 'named_references.ast', File.join(name,"array_formulae1.ast")
|
259
|
+
|
260
|
+
r = ReplaceTableReferences.new
|
261
|
+
r.sheet_name = name
|
262
|
+
replace r, File.join(name,'array_formulae1.ast'), 'all_tables', File.join(name,"array_formulae2.ast")
|
263
|
+
replace SimplifyArithmetic, File.join(name,'array_formulae2.ast'), File.join(name,'array_formulae3.ast')
|
264
|
+
replace ReplaceRangesWithArrayLiterals, File.join(name,"array_formulae3.ast"), File.join(name,"array_formulae4.ast")
|
265
|
+
rewrite RewriteArrayFormulaeToArrays, File.join(name,"array_formulae4.ast"), File.join(name,"array_formulae5.ast")
|
266
|
+
rewrite RewriteArrayFormulae, File.join(name,'array_formulae5.ast'), File.join(name,"array_formulae-expanded.ast")
|
267
|
+
end
|
268
|
+
|
269
|
+
def combine_formulae_files(name,xml_filename)
|
270
|
+
values = File.join(name,'values.ast')
|
271
|
+
shared_formulae = File.join(name,"shared_formulae-expanded.ast")
|
272
|
+
array_formulae = File.join(name,"array_formulae-expanded.ast")
|
273
|
+
simple_formulae = File.join(name,"simple_formulae.ast-nocols")
|
274
|
+
output = File.join(name,'formulae.ast')
|
275
|
+
rewrite RewriteMergeFormulaeAndValues, values, shared_formulae, array_formulae, simple_formulae, output
|
276
|
+
end
|
277
|
+
|
278
|
+
def merge_table_files
|
279
|
+
tables = []
|
280
|
+
worksheets("Merging table files") do |name,xml_filename|
|
281
|
+
tables << File.join(name,'tables')
|
282
|
+
end
|
283
|
+
if run_in_memory
|
284
|
+
o = intermediate("all_tables")
|
285
|
+
tables.each do |t|
|
286
|
+
i = input(t)
|
287
|
+
o.print i.string
|
288
|
+
close(i)
|
289
|
+
end
|
290
|
+
close(o)
|
291
|
+
else
|
292
|
+
`sort #{tables.map { |t| " '#{File.join(intermediate_directory,t)}' "}.join} > #{File.join(intermediate_directory,'all_tables')}`
|
293
|
+
end
|
294
|
+
end
|
295
|
+
|
296
|
+
def simplify_worksheets
|
297
|
+
worksheets("Simplifying") do |name,xml_filename|
|
298
|
+
replace SimplifyArithmetic, File.join(name,'formulae.ast'), File.join(name,'formulae_simple_arithmetic.ast')
|
299
|
+
|
300
|
+
replace ReplaceSharedStrings, File.join(name,'formulae_simple_arithmetic.ast'), 'shared_strings', File.join(name,"formulae_no_shared_strings.ast")
|
301
|
+
replace ReplaceSharedStrings, File.join(name,'values.ast'), 'shared_strings', File.join(name,"values_no_shared_strings.ast")
|
302
|
+
|
303
|
+
r = ReplaceNamedReferences.new
|
304
|
+
r.sheet_name = name
|
305
|
+
replace r, File.join(name,'formulae_no_shared_strings.ast'), 'named_references.ast', File.join(name,"formulae_no_named_references.ast")
|
306
|
+
|
307
|
+
r = ReplaceTableReferences.new
|
308
|
+
r.sheet_name = name
|
309
|
+
replace r, File.join(name,'formulae_no_named_references.ast'), 'all_tables', File.join(name,"formulae_no_table_references.ast")
|
310
|
+
|
311
|
+
replace ReplaceRangesWithArrayLiterals, File.join(name,"formulae_no_table_references.ast"), File.join(name,"formulae_no_ranges.ast")
|
312
|
+
end
|
313
|
+
end
|
314
|
+
|
315
|
+
def replace_blanks
|
316
|
+
references = all_formulae("formulae_no_indirects_optimised.ast")
|
317
|
+
r = ReplaceBlanks.new
|
318
|
+
r.references = references
|
319
|
+
worksheets("Replacing blanks") do |name,xml_filename|
|
320
|
+
r.default_sheet_name = name
|
321
|
+
replace r, File.join(name,"formulae_no_indirects_optimised.ast"),File.join(name,"formulae_no_blanks.ast")
|
322
|
+
end
|
323
|
+
end
|
324
|
+
|
325
|
+
def optimise_and_replace_indirect_loop
|
326
|
+
number_of_loops = 4
|
327
|
+
1.upto(number_of_loops) do |pass|
|
328
|
+
puts "Optimise and replace indirects pass #{pass}"
|
329
|
+
start = pass == 1 ? "formulae_no_ranges.ast" : "optimse-output-#{pass-1}.ast"
|
330
|
+
finish = pass == number_of_loops ? "formulae_no_indirects_optimised.ast" : "optimse-output-#{pass}.ast"
|
331
|
+
replace_indirects(start,"replace-indirect-output-#{pass}.ast","replace-indirect-working-#{pass}-")
|
332
|
+
optimise_sheets("replace-indirect-output-#{pass}.ast",finish,"optimse-working-#{pass}-")
|
333
|
+
end
|
334
|
+
end
|
335
|
+
|
336
|
+
def replace_indirects(start_filename,finish_filename,basename)
|
337
|
+
worksheets("Replacing indirects") do |name,xml_filename|
|
338
|
+
counter = 1
|
339
|
+
replace ReplaceIndirectsWithReferences, File.join(name,start_filename), File.join(name,"#{basename}#{counter+1}.ast")
|
340
|
+
counter += 1
|
341
|
+
|
342
|
+
r = ReplaceNamedReferences.new
|
343
|
+
r.sheet_name = name
|
344
|
+
replace r, File.join(name,"#{basename}#{counter}.ast"), 'named_references.ast', File.join(name,"#{basename}#{counter+1}.ast")
|
345
|
+
counter += 1
|
346
|
+
|
347
|
+
r = ReplaceTableReferences.new
|
348
|
+
r.sheet_name = name
|
349
|
+
replace r, File.join(name,"#{basename}#{counter}.ast"), 'all_tables', File.join(name,"#{basename}#{counter+1}.ast")
|
350
|
+
counter += 1
|
351
|
+
|
352
|
+
replace ReplaceRangesWithArrayLiterals, File.join(name,"#{basename}#{counter}.ast"), File.join(name,"#{basename}#{counter+1}.ast")
|
353
|
+
counter += 1
|
354
|
+
replace ReplaceArraysWithSingleCells, File.join(name,"#{basename}#{counter}.ast"), File.join(name,"#{basename}#{counter+1}.ast")
|
355
|
+
counter += 1
|
356
|
+
|
357
|
+
# Finally, create the output file
|
358
|
+
i = File.join(intermediate_directory,name,"#{basename}#{counter}.ast")
|
359
|
+
o = File.join(intermediate_directory,name,finish_filename)
|
360
|
+
if run_in_memory
|
361
|
+
@files[o] = @files[i]
|
362
|
+
else
|
363
|
+
`cp '#{i}' '#{o}'`
|
364
|
+
end
|
365
|
+
end
|
366
|
+
end
|
367
|
+
|
368
|
+
def optimise_sheets(start_filename,finish_filename,basename)
|
369
|
+
counter = 1
|
370
|
+
|
371
|
+
# Setup start
|
372
|
+
worksheets("Setting up for optimise -#{counter}") do |name|
|
373
|
+
i = File.join(intermediate_directory,name,start_filename)
|
374
|
+
o = File.join(intermediate_directory,name,"#{basename}#{counter}.ast")
|
375
|
+
if run_in_memory
|
376
|
+
@files[o] = @files[i]
|
377
|
+
else
|
378
|
+
`cp '#{i}' '#{o}'`
|
379
|
+
end
|
380
|
+
end
|
381
|
+
|
382
|
+
worksheets("Replacing with calculated values #{counter}-#{counter+1}") do |name,xml_filename|
|
383
|
+
replace ReplaceFormulaeWithCalculatedValues, File.join(name,"#{basename}#{counter}.ast"), File.join(name,"#{basename}#{counter+1}.ast")
|
384
|
+
end
|
385
|
+
counter += 1
|
386
|
+
Process.waitall
|
387
|
+
|
388
|
+
references = all_formulae("#{basename}#{counter}.ast")
|
389
|
+
inline_ast_decision = lambda do |sheet,cell,references|
|
390
|
+
references_to_keep = @cells_that_can_be_set_at_runtime[sheet]
|
391
|
+
if references_to_keep && (references_to_keep == :all || references_to_keep.include?(cell))
|
392
|
+
false
|
393
|
+
else
|
394
|
+
ast = references[sheet][cell]
|
395
|
+
if ast
|
396
|
+
if [:number,:string,:blank,:null,:error,:boolean_true,:boolean_false,:sheet_reference,:cell].include?(ast.first)
|
397
|
+
true
|
398
|
+
else
|
399
|
+
false
|
400
|
+
end
|
401
|
+
else
|
402
|
+
true # Always inline blanks
|
403
|
+
end
|
404
|
+
end
|
405
|
+
end
|
406
|
+
r = InlineFormulae.new
|
407
|
+
r.references = references
|
408
|
+
r.inline_ast = inline_ast_decision
|
409
|
+
|
410
|
+
worksheets("Inlining formulae #{counter}-#{counter+1}") do |name,xml_filename|
|
411
|
+
r.default_sheet_name = name
|
412
|
+
replace r, File.join(name,"#{basename}#{counter}.ast"), File.join(name,"#{basename}#{counter+1}.ast")
|
413
|
+
end
|
414
|
+
counter += 1
|
415
|
+
Process.waitall
|
416
|
+
|
417
|
+
# Finish
|
418
|
+
worksheets("Moving sheets #{counter}-") do |name|
|
419
|
+
o = File.join(intermediate_directory,name,finish_filename)
|
420
|
+
i = File.join(intermediate_directory,name,"#{basename}#{counter}.ast")
|
421
|
+
if run_in_memory
|
422
|
+
@files[o] = @files[i]
|
423
|
+
else
|
424
|
+
`cp '#{i}' '#{o}'`
|
425
|
+
end
|
426
|
+
end
|
427
|
+
end
|
428
|
+
|
429
|
+
def remove_any_cells_not_needed_for_outputs(formula_in = "formulae_no_blanks.ast", formula_out = "formulae_pruned.ast", values_in = "values_no_shared_strings.ast", values_out = "values_pruned.ast")
|
430
|
+
if cells_to_keep && !cells_to_keep.empty?
|
431
|
+
identifier = IdentifyDependencies.new
|
432
|
+
identifier.references = all_formulae(formula_in)
|
433
|
+
cells_to_keep.each do |sheet_to_keep,cells_to_keep|
|
434
|
+
if cells_to_keep == :all
|
435
|
+
identifier.add_depedencies_for(sheet_to_keep)
|
436
|
+
elsif cells_to_keep.is_a?(Array)
|
437
|
+
cells_to_keep.each do |cell|
|
438
|
+
identifier.add_depedencies_for(sheet_to_keep,cell)
|
439
|
+
end
|
440
|
+
end
|
441
|
+
end
|
442
|
+
r = RemoveCells.new
|
443
|
+
worksheets("Removing cells") do |name,xml_filename|
|
444
|
+
r.cells_to_keep = identifier.dependencies[name]
|
445
|
+
rewrite r, File.join(name, formula_in), File.join(name, formula_out)
|
446
|
+
rewrite r, File.join(name, values_in), File.join(name, values_out)
|
447
|
+
end
|
448
|
+
else
|
449
|
+
worksheets do |name,xml_filename|
|
450
|
+
i = File.join(intermediate_directory,name, formula_in)
|
451
|
+
o = File.join(intermediate_directory,name, formula_out)
|
452
|
+
if run_in_memory
|
453
|
+
@files[o] = @files[i]
|
454
|
+
else
|
455
|
+
`cp '#{i}' '#{o}'`
|
456
|
+
end
|
457
|
+
i = File.join(intermediate_directory,name, values_in)
|
458
|
+
o = File.join(intermediate_directory,name, values_out)
|
459
|
+
if run_in_memory
|
460
|
+
@files[o] = @files[i]
|
461
|
+
else
|
462
|
+
`cp '#{i}' '#{o}'`
|
463
|
+
end
|
464
|
+
end
|
465
|
+
end
|
466
|
+
end
|
467
|
+
|
468
|
+
def inline_formulae_that_are_only_used_once
|
469
|
+
references = all_formulae("formulae_pruned.ast")
|
470
|
+
counter = CountFormulaReferences.new
|
471
|
+
count = counter.count(references)
|
472
|
+
|
473
|
+
inline_ast_decision = lambda do |sheet,cell,references|
|
474
|
+
references_to_keep = @cells_that_can_be_set_at_runtime[sheet]
|
475
|
+
if references_to_keep && (references_to_keep == :all || references_to_keep.include?(cell))
|
476
|
+
false
|
477
|
+
else
|
478
|
+
count[sheet][cell] == 1
|
479
|
+
end
|
480
|
+
end
|
481
|
+
|
482
|
+
r = InlineFormulae.new
|
483
|
+
r.references = references
|
484
|
+
r.inline_ast = inline_ast_decision
|
485
|
+
|
486
|
+
worksheets("Inlining formulae") do |name,xml_filename|
|
487
|
+
r.default_sheet_name = name
|
488
|
+
replace r, File.join(name,"formulae_pruned.ast"), File.join(name,"formulae_inlined.ast")
|
489
|
+
end
|
490
|
+
|
491
|
+
remove_any_cells_not_needed_for_outputs("formulae_inlined.ast", "formulae_inlined_pruned.ast", "values_pruned.ast", "values_pruned2.ast")
|
492
|
+
end
|
493
|
+
|
494
|
+
def separate_formulae_elements
|
495
|
+
# First we add the sheet to all references, so that we can then look for common elements accross worksheets
|
496
|
+
r = RewriteCellReferencesToIncludeSheet.new
|
497
|
+
worksheets("Adding the sheet to all references") do |name,xml_filename|
|
498
|
+
r.worksheet = name
|
499
|
+
rewrite r, File.join(name,"formulae_inlined_pruned.ast"), File.join(name,"formulae_inlined_pruned_with_sheets.ast")
|
500
|
+
end
|
501
|
+
|
502
|
+
references = all_formulae("formulae_inlined_pruned_with_sheets.ast")
|
503
|
+
identifier = IdentifyRepeatedFormulaElements.new
|
504
|
+
repeated_elements = identifier.count(references)
|
505
|
+
repeated_elements.delete_if do |element,count|
|
506
|
+
count < 2
|
507
|
+
end
|
508
|
+
o = intermediate('common-elements-1.ast')
|
509
|
+
i = 0
|
510
|
+
repeated_elements.each do |element,count|
|
511
|
+
o.puts "common#{i}\t#{element}"
|
512
|
+
i = i + 1
|
513
|
+
end
|
514
|
+
close(o)
|
515
|
+
|
516
|
+
worksheets("Replacing repeated elements") do |name,xml_filename|
|
517
|
+
replace ReplaceCommonElementsInFormulae, File.join(name,"formulae_inlined_pruned_with_sheets.ast"), "common-elements-1.ast", File.join(name,"formulae_inlined_pruned_replaced-1.ast")
|
518
|
+
end
|
519
|
+
|
520
|
+
r = ReplaceValuesWithConstants.new
|
521
|
+
worksheets("Replacing values with constants") do |name,xml_filename|
|
522
|
+
i = input(name,"formulae_inlined_pruned_replaced-1.ast")
|
523
|
+
o = intermediate(name,"formulae_inlined_pruned_replaced.ast")
|
524
|
+
r.replace(i,o)
|
525
|
+
close(i,o)
|
526
|
+
end
|
527
|
+
|
528
|
+
puts "Replacing values with constants in common elements"
|
529
|
+
i = input("common-elements-1.ast")
|
530
|
+
o = intermediate("common-elements.ast")
|
531
|
+
r.replace(i,o)
|
532
|
+
close(i,o)
|
533
|
+
|
534
|
+
puts "Writing out constants"
|
535
|
+
co = intermediate("value_constants.ast")
|
536
|
+
r.rewriter.constants.each do |ast,constant|
|
537
|
+
co.puts "#{constant}\t#{ast}"
|
538
|
+
end
|
539
|
+
close(co)
|
540
|
+
end
|
541
|
+
|
542
|
+
# UTILITY FUNCTIONS
|
543
|
+
|
544
|
+
def settable(name)
|
545
|
+
settable_refs = @cells_that_can_be_set_at_runtime[name]
|
546
|
+
if settable_refs
|
547
|
+
lambda { |ref| (settable_refs == :all) ? true : settable_refs.include?(ref) }
|
548
|
+
else
|
549
|
+
lambda { |ref| false }
|
550
|
+
end
|
551
|
+
end
|
552
|
+
|
553
|
+
def gettable(name)
|
554
|
+
if @cells_to_keep
|
555
|
+
gettable_refs = @cells_to_keep[name]
|
556
|
+
if gettable_refs
|
557
|
+
lambda { |ref| (gettable_refs == :all) ? true : gettable_refs.include?(ref) }
|
558
|
+
else
|
559
|
+
lambda { |ref| false }
|
560
|
+
end
|
561
|
+
else
|
562
|
+
lambda { |ref| true }
|
563
|
+
end
|
564
|
+
end
|
565
|
+
|
566
|
+
def all_formulae(filename)
|
567
|
+
references = {}
|
568
|
+
worksheets do |name,xml_filename|
|
569
|
+
r = references[name] = {}
|
570
|
+
i = input(name,filename)
|
571
|
+
i.lines do |line|
|
572
|
+
line =~ /^(.*?)\t(.*)$/
|
573
|
+
ref, ast = $1, $2
|
574
|
+
r[$1] = eval($2)
|
575
|
+
end
|
576
|
+
end
|
577
|
+
references
|
578
|
+
end
|
579
|
+
|
580
|
+
def c_name_for_worksheet_name(name)
|
581
|
+
unless @worksheet_names
|
582
|
+
w = input("worksheet_c_names")
|
583
|
+
@worksheet_names = Hash[w.readlines.map { |line| line.split("\t").map { |a| a.strip }}]
|
584
|
+
close(w)
|
585
|
+
end
|
586
|
+
@worksheet_names[name]
|
587
|
+
end
|
588
|
+
|
589
|
+
def worksheets(message = "Processing",&block)
|
590
|
+
input('worksheet_names').lines.each do |line|
|
591
|
+
name, filename = *line.split("\t")
|
592
|
+
filename = File.expand_path(File.join(xml_directory,'xl',filename.strip))
|
593
|
+
puts "#{message} #{name}"
|
594
|
+
block.call(name, filename)
|
595
|
+
end
|
596
|
+
end
|
597
|
+
|
598
|
+
def extract(_klass,xml_name,output_name)
|
599
|
+
i = xml_name.is_a?(String) ? xml(xml_name) : xml_name
|
600
|
+
o = output_name.is_a?(String) ? intermediate(output_name) : output_name
|
601
|
+
_klass.extract(i,o)
|
602
|
+
if xml_name.is_a?(String)
|
603
|
+
close(i)
|
604
|
+
end
|
605
|
+
if output_name.is_a?(String)
|
606
|
+
close(o)
|
607
|
+
end
|
608
|
+
end
|
609
|
+
|
610
|
+
def rewrite(_klass,*args)
|
611
|
+
o = intermediate(args.pop)
|
612
|
+
inputs = args.map { |name| input(name) }
|
613
|
+
_klass.rewrite(*inputs,o)
|
614
|
+
close(*inputs,o)
|
615
|
+
end
|
616
|
+
|
617
|
+
def replace(_klass,*args)
|
618
|
+
o = intermediate(args.pop)
|
619
|
+
inputs = args.map { |name| input(name) }
|
620
|
+
_klass.replace(*inputs,o)
|
621
|
+
close(*inputs,o)
|
622
|
+
end
|
623
|
+
|
624
|
+
def xml(*args)
|
625
|
+
File.open(File.join(xml_directory,'xl',*args),'r')
|
626
|
+
end
|
627
|
+
|
628
|
+
def input(*args)
|
629
|
+
filename = File.join(intermediate_directory,*args)
|
630
|
+
if run_in_memory
|
631
|
+
io = StringIO.new(@files[filename].string,'r')
|
632
|
+
else
|
633
|
+
File.open(filename,'r')
|
634
|
+
end
|
635
|
+
end
|
636
|
+
|
637
|
+
def intermediate(*args)
|
638
|
+
filename = File.join(intermediate_directory,*args)
|
639
|
+
if run_in_memory
|
640
|
+
@files ||= {}
|
641
|
+
@files[filename] = StringIO.new("",'w')
|
642
|
+
else
|
643
|
+
FileUtils.mkdir_p(File.dirname(filename))
|
644
|
+
File.open(filename,'w')
|
645
|
+
end
|
646
|
+
end
|
647
|
+
|
648
|
+
def output(*args)
|
649
|
+
File.open(File.join(output_directory,*args),'w')
|
650
|
+
end
|
651
|
+
|
652
|
+
def close(*args)
|
653
|
+
args.map do |f|
|
654
|
+
next if f.is_a?(StringIO)
|
655
|
+
next if f.is_a?(String)
|
656
|
+
f.close
|
657
|
+
end
|
658
|
+
end
|
659
|
+
|
660
|
+
end
|