twb 3.7.5 → 3.9.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/lib/twb.rb +4 -1
- data/lib/twb/analysis/CalculatedFields/CalculatedFieldsAnalyzer.rb +355 -337
- data/lib/twb/analysis/CalculatedFields/MarkdownEmitter.rb +6 -4
- data/lib/twb/analysis/CalculatedFields/groupfieldsanalyzer.rb +93 -0
- data/lib/twb/analysis/DataSources/googlesheetdatasourcesanalyzer.rb +18 -9
- data/lib/twb/analysis/Sheets/analyzeDashboardSheets.rb +50 -0
- data/lib/twb/analysis/Sheets/dashsheetsanalyzer.rb +75 -0
- data/lib/twb/analysis/Sheets/sheetfieldsanalyzer.rb +20 -6
- data/lib/twb/analysis/Sheets/sheetfiltersanalyzer.rb +210 -52
- data/lib/twb/calculatedfield.rb +42 -4
- data/lib/twb/codedfield.rb +14 -13
- data/lib/twb/datasource.rb +4 -0
- data/lib/twb/tabtool.rb +60 -44
- data/lib/twb/util/fielddomainloader.rb +3 -1
- data/lib/twb/worksheet.rb +16 -1
- metadata +5 -2
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA1:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: 01e007d30bd02da639fec0bef95e94e39bf4c51b
|
|
4
|
+
data.tar.gz: 6f07a8ac8308a197494e6d89211f9569e1d31884
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 1c9413d8c790ebc4f11396b3c118a03e9bef3914b4a19b53384cf374572b24521f4f5de8cc6a7ebbc04d559d0ef13840263b68dd6da6566c22c69f39c3dacada
|
|
7
|
+
data.tar.gz: f0ebc5cbcf4e62b0b17021d97143d0b701b9af1ade87cee099c963d9af423b9cc5038c462239e9365460a69e4e0b0a144f8e52ca25c99f6852371d3e9547dd53
|
data/lib/twb.rb
CHANGED
|
@@ -48,6 +48,7 @@ require_relative 'twb/util/docprep'
|
|
|
48
48
|
require_relative 'twb/analysis/documentedfieldsmarkdownemitter'
|
|
49
49
|
require_relative 'twb/analysis/annotatedfieldsCSVEmitter'
|
|
50
50
|
require_relative 'twb/analysis/calculatedfields/calculatedfieldsanalyzer'
|
|
51
|
+
require_relative 'twb/analysis/calculatedfields/groupfieldsanalyzer'
|
|
51
52
|
require_relative 'twb/analysis/calculatedfields/markdownemitter'
|
|
52
53
|
require_relative 'twb/analysis/calculatedfields/csvemitter'
|
|
53
54
|
require_relative 'twb/analysis/datasources/DataSourceFieldsCSVEmitter'
|
|
@@ -56,9 +57,11 @@ require_relative 'twb/analysis/datasources/googlesheetdatasourcesanalyzer'
|
|
|
56
57
|
require_relative 'twb/analysis/Sheets/WorksheetDataStructureCSVEmitter'
|
|
57
58
|
require_relative 'twb/analysis/Sheets/sheetfiltersanalyzer'
|
|
58
59
|
require_relative 'twb/analysis/Sheets/sheetfieldsanalyzer'
|
|
60
|
+
require_relative 'twb/analysis/Sheets/dashsheetsanalyzer'
|
|
61
|
+
|
|
59
62
|
|
|
60
63
|
# Represents Tableau Workbooks and their contents.
|
|
61
64
|
#
|
|
62
65
|
module Twb
|
|
63
|
-
VERSION = '3.
|
|
66
|
+
VERSION = '3.9.3'
|
|
64
67
|
end
|
|
@@ -26,7 +26,7 @@ module Analysis
|
|
|
26
26
|
include TabTool
|
|
27
27
|
include Graph
|
|
28
28
|
|
|
29
|
-
attr_reader :calculatedFieldsCount, :
|
|
29
|
+
attr_reader :calculatedFieldsCount, :referencedFieldsCount, :metrics
|
|
30
30
|
attr_accessor :ttdocdir
|
|
31
31
|
|
|
32
32
|
@@ttlogfile = 'CalculatedFieldsAnalyzer.ttlog'
|
|
@@ -57,16 +57,16 @@ module Analysis
|
|
|
57
57
|
'Formula', 'Formula Line #', 'Formula Line'
|
|
58
58
|
]
|
|
59
59
|
|
|
60
|
-
@@formFieldsCSVFileName = '
|
|
60
|
+
@@formFieldsCSVFileName = 'TwbCalculatedFieldsReferenced.csv'
|
|
61
61
|
@@formFieldsCSVFileHeader = ['Rec #',
|
|
62
62
|
'Workbook', 'Workbook Dir',
|
|
63
63
|
'Data Source',
|
|
64
64
|
'Field - Calculated',
|
|
65
|
-
'
|
|
66
|
-
'
|
|
67
|
-
'Field -
|
|
68
|
-
'Field -
|
|
69
|
-
'Data Source + Field -
|
|
65
|
+
'Formula (tech)',
|
|
66
|
+
'Formula',
|
|
67
|
+
'Field - Referenced (tech)',
|
|
68
|
+
'Field - Referenced',
|
|
69
|
+
'Data Source + Field - Referenced',
|
|
70
70
|
'Table'
|
|
71
71
|
]
|
|
72
72
|
|
|
@@ -82,7 +82,8 @@ module Analysis
|
|
|
82
82
|
DOTHEADER
|
|
83
83
|
|
|
84
84
|
def initialize
|
|
85
|
-
|
|
85
|
+
init
|
|
86
|
+
@funcdoc = {:class=>self.class, :blurb=>'Analyze Calculated Fields from Tableau Workbooks.', :description=>'Calculated fields can be complex, this tool provides robust coverage.',}
|
|
86
87
|
#-- CSV records collectors
|
|
87
88
|
@csvCalculatedFields = []
|
|
88
89
|
@csvFormulaFields = []
|
|
@@ -90,7 +91,7 @@ DOTHEADER
|
|
|
90
91
|
#-- Counters setup --
|
|
91
92
|
@twbCount = 0
|
|
92
93
|
@calculatedFieldsCount = 0
|
|
93
|
-
@
|
|
94
|
+
@referencedFieldsCount = 0
|
|
94
95
|
#--
|
|
95
96
|
@referencedFields = SortedSet.new
|
|
96
97
|
#--
|
|
@@ -127,342 +128,359 @@ DOTHEADER
|
|
|
127
128
|
mapTwb
|
|
128
129
|
emitGml
|
|
129
130
|
@twbCount += 1
|
|
131
|
+
finis
|
|
130
132
|
end
|
|
131
133
|
|
|
132
|
-
def
|
|
133
|
-
|
|
134
|
-
gml.fileName = @twb.name
|
|
135
|
-
gml.nodes = @nodes
|
|
136
|
-
gml.edges = @edges
|
|
137
|
-
gml.render
|
|
138
|
-
end
|
|
139
|
-
|
|
140
|
-
def processDataSource ds
|
|
141
|
-
emit "======= DATA SOURCE: #{ds.uiname} ====== "
|
|
142
|
-
dsNodes = Set.new
|
|
143
|
-
dsEdges = Set.new
|
|
144
|
-
dsFields = {}
|
|
145
|
-
@twbFields[ds.uiname] = dsFields
|
|
146
|
-
calculatedFields = SortedSet.new
|
|
147
|
-
fieldFormulaLines = []
|
|
148
|
-
referencedFields = SortedSet.new
|
|
149
|
-
dataSourceNode = Twb::Util::Graphnode.new(name: ds.uiname, id: ds.id, type: ds, properties: {workbook: @twb.name})
|
|
150
|
-
@nodes.add dataSourceNode
|
|
151
|
-
#-- process Calculatred Fields
|
|
152
|
-
ds.calculatedFields.each do |calcField|
|
|
153
|
-
calculatedFields.add calcField.id
|
|
154
|
-
dsFields[calcField.uiname] = calcField
|
|
155
|
-
calcFieldNode = Twb::Util::Graphnode.new(name: calcField.uiname, id: calcField.id, type: calcField, properties: {:DataSource => ds.uiname})
|
|
156
|
-
@nodes.add calcFieldNode
|
|
157
|
-
dsFieldEdge = Twb::Util::Graphedge.new(from: dataSourceNode, to: calcFieldNode, relationship: 'contains')
|
|
158
|
-
@edges.add dsFieldEdge
|
|
159
|
-
calculation = calcField.calculation
|
|
160
|
-
if calculation.has_formula
|
|
161
|
-
#-- collect field formulas as single lines
|
|
162
|
-
@csvCalculatedFields.push [
|
|
163
|
-
@calculatedFieldsCount += 1,
|
|
164
|
-
@twb.name,
|
|
165
|
-
@twbDir,
|
|
166
|
-
ds.uiname,
|
|
167
|
-
ds.caption,
|
|
168
|
-
ds.name,
|
|
169
|
-
calcField.uiname,
|
|
170
|
-
calcField.caption,
|
|
171
|
-
calcField.name,
|
|
172
|
-
ds.name + '::' + calcField.name,
|
|
173
|
-
calcField.datatype,
|
|
174
|
-
calcField.role,
|
|
175
|
-
calcField.type,
|
|
176
|
-
calculation.class,
|
|
177
|
-
calculation.scopeIsolation,
|
|
178
|
-
calculation.formulaFlat.length,
|
|
179
|
-
calculation.formulaResolved,
|
|
180
|
-
calculation.formulaFlat,
|
|
181
|
-
calculation.comments,
|
|
182
|
-
calculation.is_lod
|
|
183
|
-
]
|
|
184
|
-
#-- collect individual formula lines
|
|
185
|
-
flnum = 0
|
|
186
|
-
emit "@@ FL: #{calcField.uiname}"
|
|
187
|
-
calculation.formulaResolvedLines.each do |fl|
|
|
188
|
-
emit "@@ FL: => '#{fl}'"
|
|
189
|
-
fieldFormulaLines << [ @calculatedFieldsCount, # 'Calc Field #',
|
|
190
|
-
@twb.name, # 'Workbook',
|
|
191
|
-
@twbDir, # 'Workbook Dir',
|
|
192
|
-
ds.uiname, # 'Data Source',
|
|
193
|
-
ds.caption, # 'Data Source Caption',
|
|
194
|
-
ds.name, # 'Data Source Name (tech)',
|
|
195
|
-
calcField.uiname, # 'Field Name',
|
|
196
|
-
calcField.caption, # 'Field Caption',
|
|
197
|
-
calcField.name, # 'Field Name (tech)',
|
|
198
|
-
calcField.calculation.formulaFlatResolved, # 'Formula'
|
|
199
|
-
flnum += 1, # 'Formula Line #',
|
|
200
|
-
fl.start_with?(" ") ? "'#{fl}" : fl # 'Formula Line' - THIS IS A STUPID HACK NEEDED BECAUSE TABLEAU STRIPS LEADING BLANKS FROM CSV VALUES
|
|
201
|
-
]
|
|
202
|
-
end
|
|
203
|
-
#-- collect fields referenced in formula
|
|
204
|
-
calculation.calcFields.each do |rf|
|
|
205
|
-
emit " rf.name :'#{rf.name}'"
|
|
206
|
-
emit " rf.uiname:'#{rf.uiname}'"
|
|
207
|
-
properties = {'DataSource' => ds.uiname, 'DataSourceReference' => 'local', :source => rf}
|
|
208
|
-
refFieldNode = Twb::Util::Graphnode.new(name: rf.uiname, id: rf.id, type: rf.type, properties: properties)
|
|
209
|
-
@nodes.add refFieldNode
|
|
210
|
-
fieldFieldEdge = Twb::Util::Graphedge.new(from: calcFieldNode, to: refFieldNode, relationship: 'references')
|
|
211
|
-
@edges.add fieldFieldEdge
|
|
212
|
-
referencedFields.add rf.id
|
|
213
|
-
refFieldTable = ds.fieldTable(rf.name)
|
|
214
|
-
emit "refFieldTable.nil? : #{refFieldTable.nil?}"
|
|
215
|
-
unless refFieldTable.nil?
|
|
216
|
-
tableID = refFieldTable + ':::' + ds.uiname
|
|
217
|
-
tableName = "||#{refFieldTable}||"
|
|
218
|
-
tableNode = Twb::Util::Graphnode.new(name: tableName, id: tableID, type: :DBTable, properties: properties)
|
|
219
|
-
@nodes.add tableNode
|
|
220
|
-
fieldFieldEdge = Twb::Util::Graphedge.new(from: refFieldNode, to: tableNode, relationship: 'is a field in')
|
|
221
|
-
@edges.add fieldFieldEdge
|
|
222
|
-
# fldToDsNode = tableNode
|
|
223
|
-
end
|
|
224
|
-
@csvFormulaFields << [
|
|
225
|
-
@formulaFieldsCount += 1,
|
|
226
|
-
@twb.name,
|
|
227
|
-
@twbDir,
|
|
228
|
-
ds.uiname,
|
|
229
|
-
calcField.uiname,
|
|
230
|
-
calculation.formulaFlat,
|
|
231
|
-
calculation.formulaResolved,
|
|
232
|
-
rf.name,
|
|
233
|
-
rf.uiname,
|
|
234
|
-
rf.id,
|
|
235
|
-
refFieldTable
|
|
236
|
-
]
|
|
237
|
-
end # resolvedFields.each do
|
|
238
|
-
end # if calculation.has_formula
|
|
239
|
-
end # ds.calculatedFields.each
|
|
240
|
-
|
|
241
|
-
dsRootFields = calculatedFields - referencedFields
|
|
242
|
-
@referencedFields.merge referencedFields
|
|
243
|
-
@twbRootFields.merge dsRootFields
|
|
244
|
-
cypher @twb.name
|
|
245
|
-
cypherPy @twb.name
|
|
246
|
-
emit "#######################"
|
|
247
|
-
#-- record calculated fields
|
|
248
|
-
emit "@@ record calculated fields ds: #{ds.uiname}"
|
|
249
|
-
@csvCalculatedFields.each do |r|
|
|
250
|
-
@csvCF << r
|
|
251
|
-
end
|
|
252
|
-
#-- record individual formula lines
|
|
253
|
-
emit "@@ individual formula lines ds: #{ds.uiname}"
|
|
254
|
-
fieldFormulaLines.each do |ffl|
|
|
255
|
-
@csvCFLs << ffl
|
|
256
|
-
end
|
|
257
|
-
#-- record formula-referenced fields
|
|
258
|
-
emit "@@ formula-referenced fields ds: #{ds.uiname}"
|
|
259
|
-
@csvFormulaFields.each do |r|
|
|
260
|
-
@csvFF << r
|
|
261
|
-
end
|
|
262
|
-
#--
|
|
263
|
-
return @imageFiles
|
|
264
|
-
end # def processDataSource
|
|
265
|
-
|
|
266
|
-
def emitCalcfield calcField
|
|
267
|
-
emit "\t FIELD cap :: #{calcField.caption} "
|
|
268
|
-
emit "\t tname:: #{calcField.name}"
|
|
269
|
-
emit "\t uiname:: #{calcField.uiname}"
|
|
270
|
-
emit "\t formula:: #{calculation.formulaFlat}"
|
|
271
|
-
end
|
|
272
|
-
|
|
273
|
-
def mapTwb
|
|
274
|
-
twb = @twb.name
|
|
275
|
-
rootFields = @twbRootFields
|
|
276
|
-
dotStuff = initDot twb
|
|
277
|
-
dotFile = dotStuff[:file]
|
|
278
|
-
dotFileName = dotStuff[:name]
|
|
279
|
-
dotFile.puts "\n // subgraph cluster_1 {"
|
|
280
|
-
dotFile.puts " // color= grey;"
|
|
281
|
-
dotFile.puts ""
|
|
282
|
-
edgesAsStrings = SortedSet.new
|
|
283
|
-
# this two step process coalesces the edges into a unique set, avoiding duplicating the dot
|
|
284
|
-
# file entries, and can be shrunk when graph edges expose the bits necessary for management by Set
|
|
285
|
-
emit "\n========================\nLoading Edges\n========================\n From DC? Referenced? Edge \n %s %s %s" % ['--------', '-----------', '-'*45]
|
|
286
|
-
@edges.each do |e|
|
|
287
|
-
# don't want to emit edge which is from a Data Connection to a
|
|
288
|
-
# Calculated Field which is also referenced by another calculated field
|
|
289
|
-
isFromDC = e.from.type == :TwbDataConnection
|
|
290
|
-
isRefField = @referencedFields.include?(e.to.id)
|
|
291
|
-
edgesAsStrings.add(e.dot) unless isFromDC && isRefField
|
|
292
|
-
# emit " ES #{e.dot}"
|
|
293
|
-
# emit " ES from #{e.from}"
|
|
294
|
-
# emit " ES to #{e.to}"
|
|
295
|
-
end
|
|
296
|
-
emit "------------------------\n "
|
|
297
|
-
edgesAsStrings.each do |es|
|
|
298
|
-
dotFile.puts " #{es}"
|
|
299
|
-
end
|
|
300
|
-
emit "========================\n "
|
|
301
|
-
dotFile.puts ""
|
|
302
|
-
dotFile.puts " // }"
|
|
303
|
-
dotFile.puts "\n\n // 4 NODES --------------------------------------------------------------------"
|
|
304
|
-
@nodes.each do |n|
|
|
305
|
-
dotFile.puts n.dotLabel
|
|
306
|
-
end
|
|
307
|
-
dotFile.puts "\n\n // 5--------------------------------------------------------------------"
|
|
308
|
-
emitTypes( dotFile )
|
|
309
|
-
closeDot( dotFile, twb )
|
|
310
|
-
emit "Rendering DOT file - #{twb}"
|
|
311
|
-
renderDot(twb,dotFileName,'pdf')
|
|
312
|
-
renderDot(twb,dotFileName,'png')
|
|
313
|
-
renderDot(twb,dotFileName,'svg')
|
|
314
|
-
# emitEdges
|
|
315
|
-
end
|
|
316
|
-
|
|
317
|
-
def cypher twbName
|
|
318
|
-
cypher = Twb::Util::Cypher.new
|
|
319
|
-
cypher.fileName = "#{twbName}.calcFields"
|
|
320
|
-
cypher.nodes = @nodes
|
|
321
|
-
cypher.edges = @edges
|
|
322
|
-
cypher.render
|
|
323
|
-
end
|
|
324
|
-
|
|
325
|
-
def cypherPy twbName
|
|
326
|
-
cypher = Twb::Util::CypherPython.new
|
|
327
|
-
cypher.fileName = "#{twbName}.calcFields"
|
|
328
|
-
cypher.nodes = @nodes
|
|
329
|
-
cypher.edges = @edges
|
|
330
|
-
cypher.render
|
|
134
|
+
def metrics
|
|
135
|
+
@metrics ||= loadMetrics
|
|
331
136
|
end
|
|
332
137
|
|
|
333
|
-
|
|
334
|
-
|
|
335
|
-
|
|
336
|
-
|
|
337
|
-
|
|
338
|
-
|
|
339
|
-
# # cypherCode.add edge.to.cypherCreate
|
|
340
|
-
# # cypherCode.add edge.cypherCreate
|
|
341
|
-
# # end
|
|
342
|
-
# # cypherCode.each do |cc|
|
|
343
|
-
# # graphFile.puts cc
|
|
344
|
-
# # end
|
|
345
|
-
# # graphFile.puts "\nreturn *"
|
|
346
|
-
# # graphFile.close unless graphFile.nil?
|
|
347
|
-
# # @imageFiles << File.basename(graphFile)
|
|
348
|
-
# end
|
|
349
|
-
|
|
350
|
-
def emitEdges
|
|
351
|
-
emit " %-15s %s" % ['type', 'Edge']
|
|
352
|
-
emit " %-15s %s" % ['-'*15, '-'*35]
|
|
353
|
-
@edges.each do |edge|
|
|
354
|
-
emit " %-15s %s" % [edge.from.type, edge.from]
|
|
355
|
-
emit " %-15s %s" % [edge.to.type, edge.to]
|
|
356
|
-
emit "\n "
|
|
357
|
-
end
|
|
138
|
+
def loadMetrics
|
|
139
|
+
@metrics = {
|
|
140
|
+
'# of Workbooks' => @twbCount,
|
|
141
|
+
'# of Calculated Fields' => @calculatedFieldsCount,
|
|
142
|
+
'# of Referenced Fields' => @referencedFieldsCount,
|
|
143
|
+
}
|
|
358
144
|
end
|
|
359
145
|
|
|
360
|
-
|
|
361
|
-
|
|
362
|
-
|
|
363
|
-
|
|
364
|
-
|
|
365
|
-
|
|
366
|
-
|
|
367
|
-
|
|
368
|
-
|
|
369
|
-
|
|
370
|
-
|
|
371
|
-
|
|
146
|
+
#-- private methods begin here, to end of class
|
|
147
|
+
|
|
148
|
+
private
|
|
149
|
+
|
|
150
|
+
def emitGml
|
|
151
|
+
gml = Twb::Util::GML.new
|
|
152
|
+
gml.fileName = @twb.name
|
|
153
|
+
gml.nodes = @nodes
|
|
154
|
+
gml.edges = @edges
|
|
155
|
+
gml.render
|
|
156
|
+
end
|
|
157
|
+
|
|
158
|
+
def processDataSource ds
|
|
159
|
+
emit "======= DATA SOURCE: #{ds.uiname} ====== "
|
|
160
|
+
dsNodes = Set.new
|
|
161
|
+
dsEdges = Set.new
|
|
162
|
+
dsFields = {}
|
|
163
|
+
@twbFields[ds.uiname] = dsFields
|
|
164
|
+
calculatedFields = SortedSet.new
|
|
165
|
+
fieldFormulaLines = []
|
|
166
|
+
referencedFields = SortedSet.new
|
|
167
|
+
dataSourceNode = Twb::Util::Graphnode.new(name: ds.uiname, id: ds.id, type: ds, properties: {workbook: @twb.name})
|
|
168
|
+
@nodes.add dataSourceNode
|
|
169
|
+
#-- process Calculatred Fields
|
|
170
|
+
ds.calculatedFields.each do |calcField|
|
|
171
|
+
calculatedFields.add calcField.id
|
|
172
|
+
dsFields[calcField.uiname] = calcField
|
|
173
|
+
calcFieldNode = Twb::Util::Graphnode.new(name: calcField.uiname, id: calcField.id, type: calcField, properties: {:DataSource => ds.uiname})
|
|
174
|
+
@nodes.add calcFieldNode
|
|
175
|
+
dsFieldEdge = Twb::Util::Graphedge.new(from: dataSourceNode, to: calcFieldNode, relationship: 'contains')
|
|
176
|
+
@edges.add dsFieldEdge
|
|
177
|
+
calculation = calcField.calculation
|
|
178
|
+
if calculation.has_formula
|
|
179
|
+
#-- collect field formulas as single lines
|
|
180
|
+
@csvCalculatedFields.push [
|
|
181
|
+
@calculatedFieldsCount += 1,
|
|
182
|
+
@twb.name,
|
|
183
|
+
@twbDir,
|
|
184
|
+
ds.uiname,
|
|
185
|
+
ds.caption,
|
|
186
|
+
ds.name,
|
|
187
|
+
calcField.uiname,
|
|
188
|
+
calcField.caption,
|
|
189
|
+
calcField.name,
|
|
190
|
+
ds.name + '::' + calcField.name,
|
|
191
|
+
calcField.datatype,
|
|
192
|
+
calcField.role,
|
|
193
|
+
calcField.type,
|
|
194
|
+
calculation.class,
|
|
195
|
+
calculation.scopeIsolation,
|
|
196
|
+
calculation.formulaFlat.length,
|
|
197
|
+
calculation.formulaFlatResolved,
|
|
198
|
+
calculation.formulaFlat,
|
|
199
|
+
calculation.comments,
|
|
200
|
+
calculation.is_lod
|
|
201
|
+
]
|
|
202
|
+
#-- collect individual formula lines
|
|
203
|
+
flnum = 0
|
|
204
|
+
emit "@@ FL: #{calcField.uiname}"
|
|
205
|
+
calculation.formulaResolvedLines.each do |fl|
|
|
206
|
+
emit "@@ FL: => '#{fl}'"
|
|
207
|
+
fieldFormulaLines << [ @calculatedFieldsCount, # 'Calc Field #',
|
|
208
|
+
@twb.name, # 'Workbook',
|
|
209
|
+
@twbDir, # 'Workbook Dir',
|
|
210
|
+
ds.uiname, # 'Data Source',
|
|
211
|
+
ds.caption, # 'Data Source Caption',
|
|
212
|
+
ds.name, # 'Data Source Name (tech)',
|
|
213
|
+
calcField.uiname, # 'Field Name',
|
|
214
|
+
calcField.caption, # 'Field Caption',
|
|
215
|
+
calcField.name, # 'Field Name (tech)',
|
|
216
|
+
calcField.calculation.formulaFlatResolved, # 'Formula'
|
|
217
|
+
flnum += 1, # 'Formula Line #',
|
|
218
|
+
fl.start_with?(" ") ? "'#{fl}" : fl # 'Formula Line' - THIS IS A STUPID HACK NEEDED BECAUSE TABLEAU STRIPS LEADING BLANKS FROM CSV VALUES
|
|
219
|
+
]
|
|
220
|
+
end
|
|
221
|
+
#-- collect fields referenced in formula
|
|
222
|
+
calculation.calcFields.each do |rf|
|
|
223
|
+
emit " rf.name :'#{rf.name}'"
|
|
224
|
+
emit " rf.uiname:'#{rf.uiname}'"
|
|
225
|
+
properties = {'DataSource' => ds.uiname, 'DataSourceReference' => 'local', :source => rf}
|
|
226
|
+
refFieldNode = Twb::Util::Graphnode.new(name: rf.uiname, id: rf.id, type: rf.type, properties: properties)
|
|
227
|
+
@nodes.add refFieldNode
|
|
228
|
+
fieldFieldEdge = Twb::Util::Graphedge.new(from: calcFieldNode, to: refFieldNode, relationship: 'references')
|
|
229
|
+
@edges.add fieldFieldEdge
|
|
230
|
+
referencedFields.add rf.id
|
|
231
|
+
refFieldTable = ds.fieldTable(rf.name)
|
|
232
|
+
emit "refFieldTable.nil? : #{refFieldTable.nil?}"
|
|
233
|
+
unless refFieldTable.nil?
|
|
234
|
+
tableID = refFieldTable + ':::' + ds.uiname
|
|
235
|
+
tableName = "||#{refFieldTable}||"
|
|
236
|
+
tableNode = Twb::Util::Graphnode.new(name: tableName, id: tableID, type: :DBTable, properties: properties)
|
|
237
|
+
@nodes.add tableNode
|
|
238
|
+
fieldFieldEdge = Twb::Util::Graphedge.new(from: refFieldNode, to: tableNode, relationship: 'is a field in')
|
|
239
|
+
@edges.add fieldFieldEdge
|
|
240
|
+
# fldToDsNode = tableNode
|
|
241
|
+
end
|
|
242
|
+
@csvFormulaFields << [
|
|
243
|
+
@referencedFieldsCount += 1,
|
|
244
|
+
@twb.name,
|
|
245
|
+
@twbDir,
|
|
246
|
+
ds.uiname,
|
|
247
|
+
calcField.uiname,
|
|
248
|
+
calculation.formulaFlat,
|
|
249
|
+
calculation.formulaFlatResolved,
|
|
250
|
+
rf.name,
|
|
251
|
+
rf.uiname,
|
|
252
|
+
rf.id,
|
|
253
|
+
refFieldTable
|
|
254
|
+
]
|
|
255
|
+
end # resolvedFields.each do
|
|
256
|
+
end # if calculation.has_formula
|
|
257
|
+
end # ds.calculatedFields.each
|
|
258
|
+
|
|
259
|
+
dsRootFields = calculatedFields - referencedFields
|
|
260
|
+
@referencedFields.merge referencedFields
|
|
261
|
+
@twbRootFields.merge dsRootFields
|
|
262
|
+
cypher @twb.name
|
|
263
|
+
cypherPy @twb.name
|
|
264
|
+
emit "#######################"
|
|
265
|
+
#-- record calculated fields
|
|
266
|
+
emit "@@ record calculated fields ds: #{ds.uiname}"
|
|
267
|
+
@csvCalculatedFields.each do |r|
|
|
268
|
+
@csvCF << r
|
|
269
|
+
end
|
|
270
|
+
#-- record individual formula lines
|
|
271
|
+
emit "@@ individual formula lines ds: #{ds.uiname}"
|
|
272
|
+
fieldFormulaLines.each do |ffl|
|
|
273
|
+
@csvCFLs << ffl
|
|
274
|
+
end
|
|
275
|
+
#-- record formula-referenced fields
|
|
276
|
+
emit "@@ formula-referenced fields ds: #{ds.uiname}"
|
|
277
|
+
@csvFormulaFields.each do |r|
|
|
278
|
+
@csvFF << r
|
|
279
|
+
end
|
|
280
|
+
#--
|
|
281
|
+
return @imageFiles
|
|
282
|
+
end # def processDataSource
|
|
283
|
+
|
|
284
|
+
def emitCalcfield calcField
|
|
285
|
+
emit "\t FIELD cap :: #{calcField.caption} "
|
|
286
|
+
emit "\t tname:: #{calcField.name}"
|
|
287
|
+
emit "\t uiname:: #{calcField.uiname}"
|
|
288
|
+
emit "\t formula:: #{calculation.formulaFlat}"
|
|
289
|
+
end
|
|
290
|
+
|
|
291
|
+
def mapTwb
|
|
292
|
+
twb = @twb.name
|
|
293
|
+
rootFields = @twbRootFields
|
|
294
|
+
dotStuff = initDot twb
|
|
295
|
+
dotFile = dotStuff[:file]
|
|
296
|
+
dotFileName = dotStuff[:name]
|
|
297
|
+
dotFile.puts "\n // subgraph cluster_1 {"
|
|
298
|
+
dotFile.puts " // color= grey;"
|
|
299
|
+
dotFile.puts ""
|
|
300
|
+
edgesAsStrings = SortedSet.new
|
|
301
|
+
# this two step process coalesces the edges into a unique set, avoiding duplicating the dot
|
|
302
|
+
# file entries, and can be shrunk when graph edges expose the bits necessary for management by Set
|
|
303
|
+
emit "\n========================\nLoading Edges\n========================\n From DC? Referenced? Edge \n %s %s %s" % ['--------', '-----------', '-'*45]
|
|
304
|
+
@edges.each do |e|
|
|
305
|
+
# don't want to emit edge which is from a Data Connection to a
|
|
306
|
+
# Calculated Field which is also referenced by another calculated field
|
|
307
|
+
isFromDC = e.from.type == :TwbDataConnection
|
|
308
|
+
isRefField = @referencedFields.include?(e.to.id)
|
|
309
|
+
edgesAsStrings.add(e.dot) unless isFromDC && isRefField
|
|
310
|
+
# emit " ES #{e.dot}"
|
|
311
|
+
# emit " ES from #{e.from}"
|
|
312
|
+
# emit " ES to #{e.to}"
|
|
313
|
+
end
|
|
314
|
+
emit "------------------------\n "
|
|
315
|
+
edgesAsStrings.each do |es|
|
|
316
|
+
dotFile.puts " #{es}"
|
|
317
|
+
end
|
|
318
|
+
emit "========================\n "
|
|
319
|
+
dotFile.puts ""
|
|
320
|
+
dotFile.puts " // }"
|
|
321
|
+
dotFile.puts "\n\n // 4 NODES --------------------------------------------------------------------"
|
|
322
|
+
@nodes.each do |n|
|
|
323
|
+
dotFile.puts n.dotLabel
|
|
324
|
+
end
|
|
325
|
+
dotFile.puts "\n\n // 5--------------------------------------------------------------------"
|
|
326
|
+
emitTypes( dotFile )
|
|
327
|
+
closeDot( dotFile, twb )
|
|
328
|
+
emit "Rendering DOT file - #{twb}"
|
|
329
|
+
renderDot(twb,dotFileName,'pdf')
|
|
330
|
+
renderDot(twb,dotFileName,'png')
|
|
331
|
+
renderDot(twb,dotFileName,'svg')
|
|
332
|
+
# emitEdges
|
|
333
|
+
end
|
|
334
|
+
|
|
335
|
+
def cypher twbName
|
|
336
|
+
cypher = Twb::Util::Cypher.new
|
|
337
|
+
cypher.fileName = "#{twbName}.calcFields"
|
|
338
|
+
cypher.nodes = @nodes
|
|
339
|
+
cypher.edges = @edges
|
|
340
|
+
cypher.render
|
|
341
|
+
end
|
|
342
|
+
|
|
343
|
+
def cypherPy twbName
|
|
344
|
+
cypher = Twb::Util::CypherPython.new
|
|
345
|
+
cypher.fileName = "#{twbName}.calcFields"
|
|
346
|
+
cypher.nodes = @nodes
|
|
347
|
+
cypher.edges = @edges
|
|
348
|
+
cypher.render
|
|
349
|
+
end
|
|
350
|
+
|
|
351
|
+
# def graphEdges twb
|
|
352
|
+
# # graphFile = File.new(twb + '.cypher', 'w')
|
|
353
|
+
# # # graphFile.puts "OKEY DOKE, graphing away"
|
|
354
|
+
# # cypherCode = Set.new
|
|
355
|
+
# # @edges.each do |edge|
|
|
356
|
+
# # cypherCode.add edge.from.cypherCreate
|
|
357
|
+
# # cypherCode.add edge.to.cypherCreate
|
|
358
|
+
# # cypherCode.add edge.cypherCreate
|
|
359
|
+
# # end
|
|
360
|
+
# # cypherCode.each do |cc|
|
|
361
|
+
# # graphFile.puts cc
|
|
362
|
+
# # end
|
|
363
|
+
# # graphFile.puts "\nreturn *"
|
|
364
|
+
# # graphFile.close unless graphFile.nil?
|
|
365
|
+
# # @imageFiles << File.basename(graphFile)
|
|
372
366
|
# end
|
|
373
|
-
rankSame(dotFile, type, nodes) unless type.eql? 'CalculatedField' # == :CalculatedField
|
|
374
|
-
end
|
|
375
|
-
# labelTypes dotFile, edges
|
|
376
|
-
end
|
|
377
367
|
|
|
378
|
-
|
|
379
|
-
|
|
380
|
-
|
|
381
|
-
|
|
382
|
-
|
|
383
|
-
|
|
384
|
-
|
|
385
|
-
|
|
386
|
-
|
|
387
|
-
|
|
388
|
-
|
|
389
|
-
|
|
390
|
-
|
|
391
|
-
|
|
392
|
-
|
|
393
|
-
|
|
394
|
-
|
|
395
|
-
|
|
396
|
-
|
|
397
|
-
|
|
398
|
-
|
|
399
|
-
|
|
400
|
-
|
|
401
|
-
|
|
402
|
-
|
|
403
|
-
|
|
404
|
-
|
|
405
|
-
|
|
406
|
-
|
|
407
|
-
|
|
408
|
-
|
|
409
|
-
|
|
410
|
-
|
|
411
|
-
|
|
412
|
-
|
|
413
|
-
|
|
414
|
-
|
|
415
|
-
|
|
416
|
-
|
|
417
|
-
|
|
418
|
-
|
|
419
|
-
|
|
420
|
-
|
|
421
|
-
|
|
422
|
-
|
|
423
|
-
|
|
424
|
-
|
|
425
|
-
|
|
426
|
-
|
|
427
|
-
|
|
428
|
-
|
|
429
|
-
|
|
430
|
-
|
|
431
|
-
|
|
432
|
-
|
|
433
|
-
|
|
434
|
-
|
|
435
|
-
|
|
436
|
-
|
|
437
|
-
|
|
438
|
-
|
|
439
|
-
|
|
440
|
-
|
|
441
|
-
|
|
442
|
-
|
|
443
|
-
|
|
444
|
-
|
|
445
|
-
|
|
446
|
-
|
|
447
|
-
|
|
448
|
-
|
|
449
|
-
|
|
450
|
-
|
|
451
|
-
|
|
452
|
-
|
|
453
|
-
|
|
454
|
-
|
|
455
|
-
|
|
456
|
-
|
|
457
|
-
|
|
458
|
-
|
|
459
|
-
|
|
460
|
-
|
|
461
|
-
|
|
462
|
-
|
|
463
|
-
|
|
464
|
-
|
|
465
|
-
|
|
368
|
+
def emitEdges
|
|
369
|
+
emit " %-15s %s" % ['type', 'Edge']
|
|
370
|
+
emit " %-15s %s" % ['-'*15, '-'*35]
|
|
371
|
+
@edges.each do |edge|
|
|
372
|
+
emit " %-15s %s" % [edge.from.type, edge.from]
|
|
373
|
+
emit " %-15s %s" % [edge.to.type, edge.to]
|
|
374
|
+
emit "\n "
|
|
375
|
+
end
|
|
376
|
+
end
|
|
377
|
+
|
|
378
|
+
def emitTypes dotFile
|
|
379
|
+
typedNodes = {}
|
|
380
|
+
dotFile.puts "\n\n // 2--------------------------------------------------------------------"
|
|
381
|
+
@edges.each do |edge|
|
|
382
|
+
emit " EDGE :: #{edge}"
|
|
383
|
+
loadNodeType typedNodes, edge.from
|
|
384
|
+
loadNodeType typedNodes, edge.to
|
|
385
|
+
end
|
|
386
|
+
typedNodes.each do |type, nodes|
|
|
387
|
+
# emit "+++++++++ typedNodes of '#{type}'' "
|
|
388
|
+
# nodes.each do |node|
|
|
389
|
+
# emit " -n- #{node}"
|
|
390
|
+
# end
|
|
391
|
+
rankSame(dotFile, type, nodes) unless type.eql? 'CalculatedField' # == :CalculatedField
|
|
392
|
+
end
|
|
393
|
+
# labelTypes dotFile, edges
|
|
394
|
+
end
|
|
395
|
+
|
|
396
|
+
def loadNodeType set, node
|
|
397
|
+
type = node.type
|
|
398
|
+
set[type] = Set.new unless set.include? type
|
|
399
|
+
set[type].add node
|
|
400
|
+
end
|
|
401
|
+
|
|
402
|
+
@@unrankedTypes = ['CalculationField']
|
|
403
|
+
def rankSame dotFile, type, nodes
|
|
404
|
+
return if @@unrankedTypes.include? type.to_s
|
|
405
|
+
@lines = SortedSet.new
|
|
406
|
+
nodes.each do |node|
|
|
407
|
+
@lines << node.id
|
|
408
|
+
end
|
|
409
|
+
dotFile.puts "\n // '#{type}' --------------------------------------------------------------------"
|
|
410
|
+
dotFile.puts "\n {rank=same "
|
|
411
|
+
@lines.each do |line|
|
|
412
|
+
dotFile.puts " \"#{line}\""
|
|
413
|
+
end
|
|
414
|
+
dotFile.puts " }"
|
|
415
|
+
end
|
|
416
|
+
|
|
417
|
+
def rankRootFields dotFile, dsRootFields
|
|
418
|
+
dotFile.puts "\n // Unreferenced (root) Calculated Fields -----------------------------------------"
|
|
419
|
+
dotFile.puts "\n {rank=same "
|
|
420
|
+
dsRootFields.each do |rf|
|
|
421
|
+
emit "ROOT FIELD: #{rf.class} :: #{rf}"
|
|
422
|
+
dotFile.puts " \"#{rf}\""
|
|
423
|
+
end
|
|
424
|
+
dotFile.puts " }"
|
|
425
|
+
end
|
|
426
|
+
|
|
427
|
+
|
|
428
|
+
def labelTypes dotFile
|
|
429
|
+
fromTos = Set.new
|
|
430
|
+
@edges.each do |edge|
|
|
431
|
+
# fromTos.add "\"Alien Data Source\" -> \"Alien Data Source\""
|
|
432
|
+
fromTos.add "\"#{edge.from.type}\""
|
|
433
|
+
fromTos.add "\"#{edge.to.type}\""
|
|
434
|
+
end
|
|
435
|
+
return if fromTos.empty?
|
|
436
|
+
dotFile.puts "\n // 3--------------------------------------------------------------------"
|
|
437
|
+
dotFile.puts ' subgraph cluster_0 {'
|
|
438
|
+
dotFile.puts ' color=white;'
|
|
439
|
+
dotFile.puts ' node [shape="box3d" style="filled" ];'
|
|
440
|
+
fromTos.each do |ft|
|
|
441
|
+
dotFile.puts " #{ft}"
|
|
442
|
+
end
|
|
443
|
+
dotFile.puts ' }'
|
|
444
|
+
end
|
|
445
|
+
|
|
446
|
+
def initDot twb
|
|
447
|
+
dotFileName = docFile("#{twb}#{@@processName}.dot")
|
|
448
|
+
dotFile = File.open(dotFileName,'w')
|
|
449
|
+
dotFile.puts @@dotHeader
|
|
450
|
+
return {:file => dotFile, :name => dotFileName}
|
|
451
|
+
end
|
|
452
|
+
|
|
453
|
+
def closeDot dotFile, twb
|
|
454
|
+
dotFile.puts ' '
|
|
455
|
+
dotFile.puts '// -------------------------------------------------------------'
|
|
456
|
+
dotFile.puts ' '
|
|
457
|
+
dotFile.puts ' subgraph cluster_1 {'
|
|
458
|
+
# dotFile.puts ' color=white;'
|
|
459
|
+
dotFile.puts ' style=invis;'
|
|
460
|
+
# dotFile.puts ' border=0;'
|
|
461
|
+
dotFile.puts ' node [border=blue];'
|
|
462
|
+
dotFile.puts ' '
|
|
463
|
+
dotFile.puts ' "" [style=invis]'
|
|
464
|
+
dotFile.puts " \"Tableau Tools\\nCalculated Fields Map\\nWorkbook '#{twb}'\\n#{Time.new.ctime}\" [penwidth=0]"
|
|
465
|
+
# dotFile.puts " \"Tableau Tools Workbook Calculated Fields Map\\n#{Time.new.ctime}\" -> \"\" [style=invis]"
|
|
466
|
+
dotFile.puts ' '
|
|
467
|
+
dotFile.puts ' }'
|
|
468
|
+
dotFile.puts ' '
|
|
469
|
+
dotFile.puts '}'
|
|
470
|
+
dotFile.close
|
|
471
|
+
end
|
|
472
|
+
|
|
473
|
+
|
|
474
|
+
def renderDot twb, dot, format
|
|
475
|
+
imageType = '-T' + format
|
|
476
|
+
imageFile = './ttdoc/' + twb + @@processName + 'Graph.' + format
|
|
477
|
+
imageParam = '-o"' + imageFile + '"'
|
|
478
|
+
emit "system #{@@gvDotLocation} #{imageType} #{imageParam} \"#{dot}\""
|
|
479
|
+
system "#{@@gvDotLocation} #{imageType} #{imageParam} \"#{dot}\""
|
|
480
|
+
emit " - #{imageFile}"
|
|
481
|
+
@imageFiles << imageFile
|
|
482
|
+
return imageFile
|
|
483
|
+
end
|
|
466
484
|
|
|
467
485
|
end # class
|
|
468
486
|
|