excel_to_code 0.0.1 → 0.0.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/bin/excel_to_c +11 -7
- data/src/commands.rb +1 -0
- data/src/commands/excel_to_c.rb +7 -615
- data/src/commands/excel_to_ruby.rb +14 -514
- data/src/commands/excel_to_x.rb +660 -0
- metadata +7 -6
@@ -0,0 +1,660 @@
|
|
1
|
+
# coding: utf-8
|
2
|
+
require 'fileutils'
|
3
|
+
require_relative '../excel_to_code'
|
4
|
+
|
5
|
+
# Used to throw normally fatal errors
|
6
|
+
class ExcelToCodeException < Exception; end
|
7
|
+
|
8
|
+
class ExcelToX
|
9
|
+
|
10
|
+
# Required attribute. The source excel file. This must be .xlsx not .xls
|
11
|
+
attr_accessor :excel_file
|
12
|
+
|
13
|
+
# Optional attribute. The output directory.
|
14
|
+
# If not specified, will be '#{excel_file_name}/c'
|
15
|
+
attr_accessor :output_directory
|
16
|
+
|
17
|
+
# Optional attribute. The name of the resulting c file (and associated ruby ffi module). Defaults to excelspreadsheet
|
18
|
+
attr_accessor :output_name
|
19
|
+
|
20
|
+
# Optional attribute. The excel file will be translated to xml and stored here.
|
21
|
+
# If not specified, will be '#{excel_file_name}/xml'
|
22
|
+
attr_accessor :xml_directory
|
23
|
+
|
24
|
+
# Optional attribute. The intermediate workings will be stored here.
|
25
|
+
# If not specified, will be '#{excel_file_name}/intermediate'
|
26
|
+
attr_accessor :intermediate_directory
|
27
|
+
|
28
|
+
# Optional attribute. Specifies which cells have setters created in the c code so their values can be altered at runtime.
|
29
|
+
# It is a hash. The keys are the sheet names. The values are either the symbol :all to specify that all cells on that sheet
|
30
|
+
# should be setable, or an array of cell names on that sheet that should be settable (e.g., A1)
|
31
|
+
attr_accessor :cells_that_can_be_set_at_runtime
|
32
|
+
|
33
|
+
# Optional attribute. Specifies which cells must appear in the final generated code.
|
34
|
+
# The default is that all cells in the original spreadsheet appear in the final code.
|
35
|
+
#
|
36
|
+
# If specified, then any cells that are not:
|
37
|
+
# * specified
|
38
|
+
# * required to calculate one of the cells that is specified
|
39
|
+
# * specified as a cell that can be set at runtime
|
40
|
+
# may be excluded from the final generated code.
|
41
|
+
#
|
42
|
+
# It is a hash. The keys are the sheet names. The values are either the symbol :all to specify that all cells on that sheet
|
43
|
+
# should be lept, or an array of cell names on that sheet that should be kept (e.g., A1)
|
44
|
+
attr_accessor :cells_to_keep
|
45
|
+
|
46
|
+
# Optional attribute. Boolean. Not relevant to all types of code output
|
47
|
+
# * true - the generated c code is compiled
|
48
|
+
# * false - the generated c code is not compiled (default, unless actuall_run_tests is specified as true)
|
49
|
+
attr_accessor :actually_compile_code
|
50
|
+
|
51
|
+
# Optional attribute. Boolean.
|
52
|
+
# * true - the generated tests are run
|
53
|
+
# * false - the generated tests are not run
|
54
|
+
attr_accessor :actually_run_tests
|
55
|
+
|
56
|
+
# Optional attribute. Boolean.
|
57
|
+
# * true - the intermediate files are not written to disk (requires a lot of memory)
|
58
|
+
# * false - the intermediate files are written to disk (default, easier to debug)
|
59
|
+
attr_accessor :run_in_memory
|
60
|
+
|
61
|
+
def set_defaults
|
62
|
+
raise ExcelToCodeException.new("No excel file has been specified") unless excel_file
|
63
|
+
|
64
|
+
self.output_directory ||= File.join(File.dirname(excel_file),File.basename(excel_file,".*"),'c')
|
65
|
+
self.xml_directory ||= File.join(File.dirname(excel_file),File.basename(excel_file,".*"),'xml')
|
66
|
+
self.intermediate_directory ||= File.join(File.dirname(excel_file),File.basename(excel_file,".*"),'intermediate')
|
67
|
+
|
68
|
+
self.output_name ||= "Excelspreadsheet"
|
69
|
+
|
70
|
+
self.cells_that_can_be_set_at_runtime ||= {}
|
71
|
+
|
72
|
+
# Make sure that all the cell names are downcase and don't have any $ in them
|
73
|
+
cells_that_can_be_set_at_runtime.keys.each do |sheet|
|
74
|
+
next unless cells_that_can_be_set_at_runtime[sheet].is_a?(Array)
|
75
|
+
cells_that_can_be_set_at_runtime[sheet] = cells_that_can_be_set_at_runtime[sheet].map { |reference| reference.gsub('$','').downcase }
|
76
|
+
end
|
77
|
+
|
78
|
+
# Make sure that all the cell names are downcase and don't have any $ in them
|
79
|
+
if cells_to_keep
|
80
|
+
cells_to_keep.keys.each do |sheet|
|
81
|
+
next unless cells_to_keep[sheet].is_a?(Array)
|
82
|
+
cells_to_keep[sheet] = cells_to_keep[sheet].map { |reference| reference.gsub('$','').downcase }
|
83
|
+
end
|
84
|
+
end
|
85
|
+
|
86
|
+
# Make sure the relevant directories exist
|
87
|
+
self.excel_file = File.expand_path(excel_file)
|
88
|
+
self.output_directory = File.expand_path(output_directory)
|
89
|
+
end
|
90
|
+
|
91
|
+
def go!
|
92
|
+
# This sorts out the attributes
|
93
|
+
set_defaults
|
94
|
+
|
95
|
+
# These turn the excel into a more accesible format
|
96
|
+
sort_out_output_directories
|
97
|
+
unzip_excel
|
98
|
+
|
99
|
+
# These get all the information out of the excel and put
|
100
|
+
# into a useful format
|
101
|
+
extract_data_from_workbook
|
102
|
+
extract_data_from_worksheets
|
103
|
+
merge_table_files
|
104
|
+
|
105
|
+
rewrite_worksheets
|
106
|
+
|
107
|
+
# These perform a series of transformations to the information
|
108
|
+
# with the intent of removing any redundant calculations
|
109
|
+
# that are in the excel
|
110
|
+
simplify_worksheets
|
111
|
+
optimise_and_replace_indirect_loop
|
112
|
+
replace_blanks
|
113
|
+
remove_any_cells_not_needed_for_outputs
|
114
|
+
inline_formulae_that_are_only_used_once
|
115
|
+
separate_formulae_elements
|
116
|
+
|
117
|
+
# This actually creates the code (implemented in subclasses)
|
118
|
+
write_code
|
119
|
+
|
120
|
+
# These compile and run the code version of the excel
|
121
|
+
compile_code
|
122
|
+
run_tests
|
123
|
+
|
124
|
+
puts
|
125
|
+
puts "The generated code is available in #{File.join(output_directory)}"
|
126
|
+
end
|
127
|
+
|
128
|
+
def sort_out_output_directories
|
129
|
+
FileUtils.mkdir_p(output_directory)
|
130
|
+
FileUtils.mkdir_p(xml_directory)
|
131
|
+
FileUtils.mkdir_p(intermediate_directory)
|
132
|
+
end
|
133
|
+
|
134
|
+
def unzip_excel
|
135
|
+
puts `rm -fr '#{xml_directory}'`
|
136
|
+
puts `unzip -uo '#{excel_file}' -d '#{xml_directory}'`
|
137
|
+
end
|
138
|
+
|
139
|
+
def extract_data_from_workbook
|
140
|
+
extract_shared_strings
|
141
|
+
extract_named_references
|
142
|
+
extract_worksheet_names
|
143
|
+
extract_dimensions_from_worksheets
|
144
|
+
end
|
145
|
+
|
146
|
+
def extract_shared_strings
|
147
|
+
if File.exists?(File.join(xml_directory,'xl','sharedStrings.xml'))
|
148
|
+
extract ExtractSharedStrings, 'sharedStrings.xml', 'shared_strings'
|
149
|
+
else
|
150
|
+
FileUtils.touch(File.join(intermediate_directory,'shared_strings'))
|
151
|
+
end
|
152
|
+
end
|
153
|
+
|
154
|
+
def extract_named_references
|
155
|
+
extract ExtractNamedReferences, 'workbook.xml', 'named_references'
|
156
|
+
rewrite RewriteFormulaeToAst, 'named_references', 'named_references.ast'
|
157
|
+
end
|
158
|
+
|
159
|
+
def extract_worksheet_names
|
160
|
+
extract ExtractWorksheetNames, 'workbook.xml', 'worksheet_names_without_filenames'
|
161
|
+
extract ExtractRelationships, File.join('_rels','workbook.xml.rels'), 'workbook_relationships'
|
162
|
+
rewrite RewriteWorksheetNames, 'worksheet_names_without_filenames', 'workbook_relationships', 'worksheet_names'
|
163
|
+
rewrite MapSheetNamesToCNames, 'worksheet_names', 'worksheet_c_names'
|
164
|
+
end
|
165
|
+
|
166
|
+
def extract_dimensions_from_worksheets
|
167
|
+
dimension_file = intermediate('dimensions')
|
168
|
+
worksheets("Extracting dimensions") do |name,xml_filename|
|
169
|
+
dimension_file.write name
|
170
|
+
dimension_file.write "\t"
|
171
|
+
extract ExtractWorksheetDimensions, File.open(xml_filename,'r'), dimension_file
|
172
|
+
end
|
173
|
+
dimension_file.close
|
174
|
+
end
|
175
|
+
|
176
|
+
def extract_data_from_worksheets
|
177
|
+
worksheets("Initial data extract") do |name,xml_filename|
|
178
|
+
worksheet_directory = File.join(intermediate_directory,name)
|
179
|
+
worksheet_xml = File.open(xml_filename,'r')
|
180
|
+
|
181
|
+
worksheet_xml.rewind
|
182
|
+
extract ExtractValues, worksheet_xml, File.join(name,'values')
|
183
|
+
rewrite RewriteValuesToAst, File.join(name,'values'), File.join(name,'values.ast')
|
184
|
+
|
185
|
+
worksheet_xml.rewind
|
186
|
+
extract ExtractSimpleFormulae, worksheet_xml, File.join(name,'simple_formulae')
|
187
|
+
rewrite RewriteFormulaeToAst, File.join(name,'simple_formulae'), File.join(name,'simple_formulae.ast')
|
188
|
+
|
189
|
+
worksheet_xml.rewind
|
190
|
+
extract ExtractSharedFormulae, worksheet_xml, File.join(name,'shared_formulae')
|
191
|
+
rewrite RewriteFormulaeToAst, File.join(name,'shared_formulae'), File.join(name,'shared_formulae.ast')
|
192
|
+
|
193
|
+
worksheet_xml.rewind
|
194
|
+
extract ExtractArrayFormulae, worksheet_xml, File.join(name,'array_formulae')
|
195
|
+
rewrite RewriteFormulaeToAst, File.join(name,'array_formulae'), File.join(name,'array_formulae.ast')
|
196
|
+
|
197
|
+
worksheet_xml.rewind
|
198
|
+
extract ExtractWorksheetTableRelationships, worksheet_xml, File.join(name,'table_rids')
|
199
|
+
if File.exists?(File.join(xml_directory,'xl','worksheets','_rels',"#{File.basename(xml_filename)}.rels"))
|
200
|
+
extract_tables(name,xml_filename)
|
201
|
+
else
|
202
|
+
fake_extract_tables(name,xml_filename)
|
203
|
+
end
|
204
|
+
close(worksheet_xml)
|
205
|
+
end
|
206
|
+
end
|
207
|
+
|
208
|
+
def extract_tables(name,xml_filename)
|
209
|
+
extract ExtractRelationships, File.join('worksheets','_rels',"#{File.basename(xml_filename)}.rels"), File.join(name,'relationships')
|
210
|
+
rewrite RewriteRelationshipIdToFilename, File.join(name,'table_rids'), File.join(name,'relationships'), File.join(name,'table_filenames')
|
211
|
+
tables = intermediate(name,'tables')
|
212
|
+
table_extractor = ExtractTable.new(name)
|
213
|
+
table_filenames = input(name,'table_filenames')
|
214
|
+
table_filenames.lines.each do |line|
|
215
|
+
extract table_extractor, File.join('worksheets',line.strip), tables
|
216
|
+
end
|
217
|
+
close(tables,table_filenames)
|
218
|
+
end
|
219
|
+
|
220
|
+
def fake_extract_tables(name,xml_filename)
|
221
|
+
a = intermediate(name,'relationships')
|
222
|
+
b = intermediate(name,'table_filenames')
|
223
|
+
c = intermediate(name,'tables')
|
224
|
+
close(a,b,c)
|
225
|
+
end
|
226
|
+
|
227
|
+
def rewrite_worksheets
|
228
|
+
worksheets("Initial rewrite of references and formulae") do |name,xml_filename|
|
229
|
+
rewrite_row_and_column_references(name,xml_filename)
|
230
|
+
rewrite_shared_formulae(name,xml_filename)
|
231
|
+
rewrite_array_formulae(name,xml_filename)
|
232
|
+
combine_formulae_files(name,xml_filename)
|
233
|
+
end
|
234
|
+
end
|
235
|
+
|
236
|
+
def rewrite_row_and_column_references(name,xml_filename)
|
237
|
+
dimensions = input('dimensions')
|
238
|
+
%w{simple_formulae.ast shared_formulae.ast array_formulae.ast}.each do |file|
|
239
|
+
dimensions.rewind
|
240
|
+
i = input(name,file)
|
241
|
+
o = intermediate(name,"#{file}-nocols")
|
242
|
+
RewriteWholeRowColumnReferencesToAreas.rewrite(i,name, dimensions, o)
|
243
|
+
close(i,o)
|
244
|
+
end
|
245
|
+
dimensions.close
|
246
|
+
end
|
247
|
+
|
248
|
+
def rewrite_shared_formulae(name,xml_filename)
|
249
|
+
i = input(name,'shared_formulae.ast-nocols')
|
250
|
+
o = intermediate(name,"shared_formulae-expanded.ast")
|
251
|
+
RewriteSharedFormulae.rewrite(i,o)
|
252
|
+
close(i,o)
|
253
|
+
end
|
254
|
+
|
255
|
+
def rewrite_array_formulae(name,xml_filename)
|
256
|
+
r = ReplaceNamedReferences.new
|
257
|
+
r.sheet_name = name
|
258
|
+
replace r, File.join(name,'array_formulae.ast-nocols'), 'named_references.ast', File.join(name,"array_formulae1.ast")
|
259
|
+
|
260
|
+
r = ReplaceTableReferences.new
|
261
|
+
r.sheet_name = name
|
262
|
+
replace r, File.join(name,'array_formulae1.ast'), 'all_tables', File.join(name,"array_formulae2.ast")
|
263
|
+
replace SimplifyArithmetic, File.join(name,'array_formulae2.ast'), File.join(name,'array_formulae3.ast')
|
264
|
+
replace ReplaceRangesWithArrayLiterals, File.join(name,"array_formulae3.ast"), File.join(name,"array_formulae4.ast")
|
265
|
+
rewrite RewriteArrayFormulaeToArrays, File.join(name,"array_formulae4.ast"), File.join(name,"array_formulae5.ast")
|
266
|
+
rewrite RewriteArrayFormulae, File.join(name,'array_formulae5.ast'), File.join(name,"array_formulae-expanded.ast")
|
267
|
+
end
|
268
|
+
|
269
|
+
def combine_formulae_files(name,xml_filename)
|
270
|
+
values = File.join(name,'values.ast')
|
271
|
+
shared_formulae = File.join(name,"shared_formulae-expanded.ast")
|
272
|
+
array_formulae = File.join(name,"array_formulae-expanded.ast")
|
273
|
+
simple_formulae = File.join(name,"simple_formulae.ast-nocols")
|
274
|
+
output = File.join(name,'formulae.ast')
|
275
|
+
rewrite RewriteMergeFormulaeAndValues, values, shared_formulae, array_formulae, simple_formulae, output
|
276
|
+
end
|
277
|
+
|
278
|
+
def merge_table_files
|
279
|
+
tables = []
|
280
|
+
worksheets("Merging table files") do |name,xml_filename|
|
281
|
+
tables << File.join(name,'tables')
|
282
|
+
end
|
283
|
+
if run_in_memory
|
284
|
+
o = intermediate("all_tables")
|
285
|
+
tables.each do |t|
|
286
|
+
i = input(t)
|
287
|
+
o.print i.string
|
288
|
+
close(i)
|
289
|
+
end
|
290
|
+
close(o)
|
291
|
+
else
|
292
|
+
`sort #{tables.map { |t| " '#{File.join(intermediate_directory,t)}' "}.join} > #{File.join(intermediate_directory,'all_tables')}`
|
293
|
+
end
|
294
|
+
end
|
295
|
+
|
296
|
+
def simplify_worksheets
|
297
|
+
worksheets("Simplifying") do |name,xml_filename|
|
298
|
+
replace SimplifyArithmetic, File.join(name,'formulae.ast'), File.join(name,'formulae_simple_arithmetic.ast')
|
299
|
+
|
300
|
+
replace ReplaceSharedStrings, File.join(name,'formulae_simple_arithmetic.ast'), 'shared_strings', File.join(name,"formulae_no_shared_strings.ast")
|
301
|
+
replace ReplaceSharedStrings, File.join(name,'values.ast'), 'shared_strings', File.join(name,"values_no_shared_strings.ast")
|
302
|
+
|
303
|
+
r = ReplaceNamedReferences.new
|
304
|
+
r.sheet_name = name
|
305
|
+
replace r, File.join(name,'formulae_no_shared_strings.ast'), 'named_references.ast', File.join(name,"formulae_no_named_references.ast")
|
306
|
+
|
307
|
+
r = ReplaceTableReferences.new
|
308
|
+
r.sheet_name = name
|
309
|
+
replace r, File.join(name,'formulae_no_named_references.ast'), 'all_tables', File.join(name,"formulae_no_table_references.ast")
|
310
|
+
|
311
|
+
replace ReplaceRangesWithArrayLiterals, File.join(name,"formulae_no_table_references.ast"), File.join(name,"formulae_no_ranges.ast")
|
312
|
+
end
|
313
|
+
end
|
314
|
+
|
315
|
+
def replace_blanks
|
316
|
+
references = all_formulae("formulae_no_indirects_optimised.ast")
|
317
|
+
r = ReplaceBlanks.new
|
318
|
+
r.references = references
|
319
|
+
worksheets("Replacing blanks") do |name,xml_filename|
|
320
|
+
r.default_sheet_name = name
|
321
|
+
replace r, File.join(name,"formulae_no_indirects_optimised.ast"),File.join(name,"formulae_no_blanks.ast")
|
322
|
+
end
|
323
|
+
end
|
324
|
+
|
325
|
+
def optimise_and_replace_indirect_loop
|
326
|
+
number_of_loops = 4
|
327
|
+
1.upto(number_of_loops) do |pass|
|
328
|
+
puts "Optimise and replace indirects pass #{pass}"
|
329
|
+
start = pass == 1 ? "formulae_no_ranges.ast" : "optimse-output-#{pass-1}.ast"
|
330
|
+
finish = pass == number_of_loops ? "formulae_no_indirects_optimised.ast" : "optimse-output-#{pass}.ast"
|
331
|
+
replace_indirects(start,"replace-indirect-output-#{pass}.ast","replace-indirect-working-#{pass}-")
|
332
|
+
optimise_sheets("replace-indirect-output-#{pass}.ast",finish,"optimse-working-#{pass}-")
|
333
|
+
end
|
334
|
+
end
|
335
|
+
|
336
|
+
def replace_indirects(start_filename,finish_filename,basename)
|
337
|
+
worksheets("Replacing indirects") do |name,xml_filename|
|
338
|
+
counter = 1
|
339
|
+
replace ReplaceIndirectsWithReferences, File.join(name,start_filename), File.join(name,"#{basename}#{counter+1}.ast")
|
340
|
+
counter += 1
|
341
|
+
|
342
|
+
r = ReplaceNamedReferences.new
|
343
|
+
r.sheet_name = name
|
344
|
+
replace r, File.join(name,"#{basename}#{counter}.ast"), 'named_references.ast', File.join(name,"#{basename}#{counter+1}.ast")
|
345
|
+
counter += 1
|
346
|
+
|
347
|
+
r = ReplaceTableReferences.new
|
348
|
+
r.sheet_name = name
|
349
|
+
replace r, File.join(name,"#{basename}#{counter}.ast"), 'all_tables', File.join(name,"#{basename}#{counter+1}.ast")
|
350
|
+
counter += 1
|
351
|
+
|
352
|
+
replace ReplaceRangesWithArrayLiterals, File.join(name,"#{basename}#{counter}.ast"), File.join(name,"#{basename}#{counter+1}.ast")
|
353
|
+
counter += 1
|
354
|
+
replace ReplaceArraysWithSingleCells, File.join(name,"#{basename}#{counter}.ast"), File.join(name,"#{basename}#{counter+1}.ast")
|
355
|
+
counter += 1
|
356
|
+
|
357
|
+
# Finally, create the output file
|
358
|
+
i = File.join(intermediate_directory,name,"#{basename}#{counter}.ast")
|
359
|
+
o = File.join(intermediate_directory,name,finish_filename)
|
360
|
+
if run_in_memory
|
361
|
+
@files[o] = @files[i]
|
362
|
+
else
|
363
|
+
`cp '#{i}' '#{o}'`
|
364
|
+
end
|
365
|
+
end
|
366
|
+
end
|
367
|
+
|
368
|
+
def optimise_sheets(start_filename,finish_filename,basename)
|
369
|
+
counter = 1
|
370
|
+
|
371
|
+
# Setup start
|
372
|
+
worksheets("Setting up for optimise -#{counter}") do |name|
|
373
|
+
i = File.join(intermediate_directory,name,start_filename)
|
374
|
+
o = File.join(intermediate_directory,name,"#{basename}#{counter}.ast")
|
375
|
+
if run_in_memory
|
376
|
+
@files[o] = @files[i]
|
377
|
+
else
|
378
|
+
`cp '#{i}' '#{o}'`
|
379
|
+
end
|
380
|
+
end
|
381
|
+
|
382
|
+
worksheets("Replacing with calculated values #{counter}-#{counter+1}") do |name,xml_filename|
|
383
|
+
replace ReplaceFormulaeWithCalculatedValues, File.join(name,"#{basename}#{counter}.ast"), File.join(name,"#{basename}#{counter+1}.ast")
|
384
|
+
end
|
385
|
+
counter += 1
|
386
|
+
Process.waitall
|
387
|
+
|
388
|
+
references = all_formulae("#{basename}#{counter}.ast")
|
389
|
+
inline_ast_decision = lambda do |sheet,cell,references|
|
390
|
+
references_to_keep = @cells_that_can_be_set_at_runtime[sheet]
|
391
|
+
if references_to_keep && (references_to_keep == :all || references_to_keep.include?(cell))
|
392
|
+
false
|
393
|
+
else
|
394
|
+
ast = references[sheet][cell]
|
395
|
+
if ast
|
396
|
+
if [:number,:string,:blank,:null,:error,:boolean_true,:boolean_false,:sheet_reference,:cell].include?(ast.first)
|
397
|
+
true
|
398
|
+
else
|
399
|
+
false
|
400
|
+
end
|
401
|
+
else
|
402
|
+
true # Always inline blanks
|
403
|
+
end
|
404
|
+
end
|
405
|
+
end
|
406
|
+
r = InlineFormulae.new
|
407
|
+
r.references = references
|
408
|
+
r.inline_ast = inline_ast_decision
|
409
|
+
|
410
|
+
worksheets("Inlining formulae #{counter}-#{counter+1}") do |name,xml_filename|
|
411
|
+
r.default_sheet_name = name
|
412
|
+
replace r, File.join(name,"#{basename}#{counter}.ast"), File.join(name,"#{basename}#{counter+1}.ast")
|
413
|
+
end
|
414
|
+
counter += 1
|
415
|
+
Process.waitall
|
416
|
+
|
417
|
+
# Finish
|
418
|
+
worksheets("Moving sheets #{counter}-") do |name|
|
419
|
+
o = File.join(intermediate_directory,name,finish_filename)
|
420
|
+
i = File.join(intermediate_directory,name,"#{basename}#{counter}.ast")
|
421
|
+
if run_in_memory
|
422
|
+
@files[o] = @files[i]
|
423
|
+
else
|
424
|
+
`cp '#{i}' '#{o}'`
|
425
|
+
end
|
426
|
+
end
|
427
|
+
end
|
428
|
+
|
429
|
+
def remove_any_cells_not_needed_for_outputs(formula_in = "formulae_no_blanks.ast", formula_out = "formulae_pruned.ast", values_in = "values_no_shared_strings.ast", values_out = "values_pruned.ast")
|
430
|
+
if cells_to_keep && !cells_to_keep.empty?
|
431
|
+
identifier = IdentifyDependencies.new
|
432
|
+
identifier.references = all_formulae(formula_in)
|
433
|
+
cells_to_keep.each do |sheet_to_keep,cells_to_keep|
|
434
|
+
if cells_to_keep == :all
|
435
|
+
identifier.add_depedencies_for(sheet_to_keep)
|
436
|
+
elsif cells_to_keep.is_a?(Array)
|
437
|
+
cells_to_keep.each do |cell|
|
438
|
+
identifier.add_depedencies_for(sheet_to_keep,cell)
|
439
|
+
end
|
440
|
+
end
|
441
|
+
end
|
442
|
+
r = RemoveCells.new
|
443
|
+
worksheets("Removing cells") do |name,xml_filename|
|
444
|
+
r.cells_to_keep = identifier.dependencies[name]
|
445
|
+
rewrite r, File.join(name, formula_in), File.join(name, formula_out)
|
446
|
+
rewrite r, File.join(name, values_in), File.join(name, values_out)
|
447
|
+
end
|
448
|
+
else
|
449
|
+
worksheets do |name,xml_filename|
|
450
|
+
i = File.join(intermediate_directory,name, formula_in)
|
451
|
+
o = File.join(intermediate_directory,name, formula_out)
|
452
|
+
if run_in_memory
|
453
|
+
@files[o] = @files[i]
|
454
|
+
else
|
455
|
+
`cp '#{i}' '#{o}'`
|
456
|
+
end
|
457
|
+
i = File.join(intermediate_directory,name, values_in)
|
458
|
+
o = File.join(intermediate_directory,name, values_out)
|
459
|
+
if run_in_memory
|
460
|
+
@files[o] = @files[i]
|
461
|
+
else
|
462
|
+
`cp '#{i}' '#{o}'`
|
463
|
+
end
|
464
|
+
end
|
465
|
+
end
|
466
|
+
end
|
467
|
+
|
468
|
+
def inline_formulae_that_are_only_used_once
|
469
|
+
references = all_formulae("formulae_pruned.ast")
|
470
|
+
counter = CountFormulaReferences.new
|
471
|
+
count = counter.count(references)
|
472
|
+
|
473
|
+
inline_ast_decision = lambda do |sheet,cell,references|
|
474
|
+
references_to_keep = @cells_that_can_be_set_at_runtime[sheet]
|
475
|
+
if references_to_keep && (references_to_keep == :all || references_to_keep.include?(cell))
|
476
|
+
false
|
477
|
+
else
|
478
|
+
count[sheet][cell] == 1
|
479
|
+
end
|
480
|
+
end
|
481
|
+
|
482
|
+
r = InlineFormulae.new
|
483
|
+
r.references = references
|
484
|
+
r.inline_ast = inline_ast_decision
|
485
|
+
|
486
|
+
worksheets("Inlining formulae") do |name,xml_filename|
|
487
|
+
r.default_sheet_name = name
|
488
|
+
replace r, File.join(name,"formulae_pruned.ast"), File.join(name,"formulae_inlined.ast")
|
489
|
+
end
|
490
|
+
|
491
|
+
remove_any_cells_not_needed_for_outputs("formulae_inlined.ast", "formulae_inlined_pruned.ast", "values_pruned.ast", "values_pruned2.ast")
|
492
|
+
end
|
493
|
+
|
494
|
+
def separate_formulae_elements
|
495
|
+
# First we add the sheet to all references, so that we can then look for common elements accross worksheets
|
496
|
+
r = RewriteCellReferencesToIncludeSheet.new
|
497
|
+
worksheets("Adding the sheet to all references") do |name,xml_filename|
|
498
|
+
r.worksheet = name
|
499
|
+
rewrite r, File.join(name,"formulae_inlined_pruned.ast"), File.join(name,"formulae_inlined_pruned_with_sheets.ast")
|
500
|
+
end
|
501
|
+
|
502
|
+
references = all_formulae("formulae_inlined_pruned_with_sheets.ast")
|
503
|
+
identifier = IdentifyRepeatedFormulaElements.new
|
504
|
+
repeated_elements = identifier.count(references)
|
505
|
+
repeated_elements.delete_if do |element,count|
|
506
|
+
count < 2
|
507
|
+
end
|
508
|
+
o = intermediate('common-elements-1.ast')
|
509
|
+
i = 0
|
510
|
+
repeated_elements.each do |element,count|
|
511
|
+
o.puts "common#{i}\t#{element}"
|
512
|
+
i = i + 1
|
513
|
+
end
|
514
|
+
close(o)
|
515
|
+
|
516
|
+
worksheets("Replacing repeated elements") do |name,xml_filename|
|
517
|
+
replace ReplaceCommonElementsInFormulae, File.join(name,"formulae_inlined_pruned_with_sheets.ast"), "common-elements-1.ast", File.join(name,"formulae_inlined_pruned_replaced-1.ast")
|
518
|
+
end
|
519
|
+
|
520
|
+
r = ReplaceValuesWithConstants.new
|
521
|
+
worksheets("Replacing values with constants") do |name,xml_filename|
|
522
|
+
i = input(name,"formulae_inlined_pruned_replaced-1.ast")
|
523
|
+
o = intermediate(name,"formulae_inlined_pruned_replaced.ast")
|
524
|
+
r.replace(i,o)
|
525
|
+
close(i,o)
|
526
|
+
end
|
527
|
+
|
528
|
+
puts "Replacing values with constants in common elements"
|
529
|
+
i = input("common-elements-1.ast")
|
530
|
+
o = intermediate("common-elements.ast")
|
531
|
+
r.replace(i,o)
|
532
|
+
close(i,o)
|
533
|
+
|
534
|
+
puts "Writing out constants"
|
535
|
+
co = intermediate("value_constants.ast")
|
536
|
+
r.rewriter.constants.each do |ast,constant|
|
537
|
+
co.puts "#{constant}\t#{ast}"
|
538
|
+
end
|
539
|
+
close(co)
|
540
|
+
end
|
541
|
+
|
542
|
+
# UTILITY FUNCTIONS
|
543
|
+
|
544
|
+
def settable(name)
|
545
|
+
settable_refs = @cells_that_can_be_set_at_runtime[name]
|
546
|
+
if settable_refs
|
547
|
+
lambda { |ref| (settable_refs == :all) ? true : settable_refs.include?(ref) }
|
548
|
+
else
|
549
|
+
lambda { |ref| false }
|
550
|
+
end
|
551
|
+
end
|
552
|
+
|
553
|
+
def gettable(name)
|
554
|
+
if @cells_to_keep
|
555
|
+
gettable_refs = @cells_to_keep[name]
|
556
|
+
if gettable_refs
|
557
|
+
lambda { |ref| (gettable_refs == :all) ? true : gettable_refs.include?(ref) }
|
558
|
+
else
|
559
|
+
lambda { |ref| false }
|
560
|
+
end
|
561
|
+
else
|
562
|
+
lambda { |ref| true }
|
563
|
+
end
|
564
|
+
end
|
565
|
+
|
566
|
+
def all_formulae(filename)
|
567
|
+
references = {}
|
568
|
+
worksheets do |name,xml_filename|
|
569
|
+
r = references[name] = {}
|
570
|
+
i = input(name,filename)
|
571
|
+
i.lines do |line|
|
572
|
+
line =~ /^(.*?)\t(.*)$/
|
573
|
+
ref, ast = $1, $2
|
574
|
+
r[$1] = eval($2)
|
575
|
+
end
|
576
|
+
end
|
577
|
+
references
|
578
|
+
end
|
579
|
+
|
580
|
+
def c_name_for_worksheet_name(name)
|
581
|
+
unless @worksheet_names
|
582
|
+
w = input("worksheet_c_names")
|
583
|
+
@worksheet_names = Hash[w.readlines.map { |line| line.split("\t").map { |a| a.strip }}]
|
584
|
+
close(w)
|
585
|
+
end
|
586
|
+
@worksheet_names[name]
|
587
|
+
end
|
588
|
+
|
589
|
+
def worksheets(message = "Processing",&block)
|
590
|
+
input('worksheet_names').lines.each do |line|
|
591
|
+
name, filename = *line.split("\t")
|
592
|
+
filename = File.expand_path(File.join(xml_directory,'xl',filename.strip))
|
593
|
+
puts "#{message} #{name}"
|
594
|
+
block.call(name, filename)
|
595
|
+
end
|
596
|
+
end
|
597
|
+
|
598
|
+
def extract(_klass,xml_name,output_name)
|
599
|
+
i = xml_name.is_a?(String) ? xml(xml_name) : xml_name
|
600
|
+
o = output_name.is_a?(String) ? intermediate(output_name) : output_name
|
601
|
+
_klass.extract(i,o)
|
602
|
+
if xml_name.is_a?(String)
|
603
|
+
close(i)
|
604
|
+
end
|
605
|
+
if output_name.is_a?(String)
|
606
|
+
close(o)
|
607
|
+
end
|
608
|
+
end
|
609
|
+
|
610
|
+
def rewrite(_klass,*args)
|
611
|
+
o = intermediate(args.pop)
|
612
|
+
inputs = args.map { |name| input(name) }
|
613
|
+
_klass.rewrite(*inputs,o)
|
614
|
+
close(*inputs,o)
|
615
|
+
end
|
616
|
+
|
617
|
+
def replace(_klass,*args)
|
618
|
+
o = intermediate(args.pop)
|
619
|
+
inputs = args.map { |name| input(name) }
|
620
|
+
_klass.replace(*inputs,o)
|
621
|
+
close(*inputs,o)
|
622
|
+
end
|
623
|
+
|
624
|
+
def xml(*args)
|
625
|
+
File.open(File.join(xml_directory,'xl',*args),'r')
|
626
|
+
end
|
627
|
+
|
628
|
+
def input(*args)
|
629
|
+
filename = File.join(intermediate_directory,*args)
|
630
|
+
if run_in_memory
|
631
|
+
io = StringIO.new(@files[filename].string,'r')
|
632
|
+
else
|
633
|
+
File.open(filename,'r')
|
634
|
+
end
|
635
|
+
end
|
636
|
+
|
637
|
+
def intermediate(*args)
|
638
|
+
filename = File.join(intermediate_directory,*args)
|
639
|
+
if run_in_memory
|
640
|
+
@files ||= {}
|
641
|
+
@files[filename] = StringIO.new("",'w')
|
642
|
+
else
|
643
|
+
FileUtils.mkdir_p(File.dirname(filename))
|
644
|
+
File.open(filename,'w')
|
645
|
+
end
|
646
|
+
end
|
647
|
+
|
648
|
+
def output(*args)
|
649
|
+
File.open(File.join(output_directory,*args),'w')
|
650
|
+
end
|
651
|
+
|
652
|
+
def close(*args)
|
653
|
+
args.map do |f|
|
654
|
+
next if f.is_a?(StringIO)
|
655
|
+
next if f.is_a?(String)
|
656
|
+
f.close
|
657
|
+
end
|
658
|
+
end
|
659
|
+
|
660
|
+
end
|