twb 1.0.5 → 1.9.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +2 -2
- data/lib/twb.rb +10 -6
- data/lib/twb/analysis/CalculatedFields/CSVEmitter.rb +154 -0
- data/lib/twb/analysis/CalculatedFields/CalculatedFieldsAnalyzer.rb +527 -0
- data/lib/twb/analysis/CalculatedFields/MarkdownEmitter.rb +71 -0
- data/lib/twb/analysis/DataSources/DataSourceTableFieldsCSVEmitter.rb +117 -0
- data/lib/twb/analysis/Sheets/WorksheetDataStructureCSVEmitter.rb +192 -0
- data/lib/twb/calculatedfield.rb +47 -0
- data/lib/twb/columnfield.rb +94 -0
- data/lib/twb/dashboard.rb +1 -1
- data/lib/twb/datasource.rb +368 -121
- data/lib/twb/fieldcalculation.rb +157 -81
- data/lib/twb/localfield.rb +32 -29
- data/lib/twb/metadatafield.rb +57 -15
- data/lib/twb/util/ftpPublisher.rb +48 -0
- data/lib/twb/util/joinUtilities.rb +52 -0
- data/lib/twb/workbook.rb +27 -10
- data/lib/twb/worksheet.rb +146 -53
- metadata +11 -5
- data/lib/twb/analysis/calculatedfieldsanalyzer.rb +0 -508
- data/lib/twb/apps/X-Ray Dashboards.rb +0 -80
- data/lib/twb/countNodes.rb +0 -98
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA1:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: 44eeddf556df0d517a68f5b4a1af89ae52138f37
|
|
4
|
+
data.tar.gz: 46d6c474f5d6d5b020d7849463733c66bfb724fe
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 6ba98c71c6cd2ccfa8bb6800e20541e90d1e8c9d8da2a420b93a8b8ba5b91912228185fd4a9d6ae20e66ea0da98b62c24b1f24cb9d631fbae9de028742c9fbef
|
|
7
|
+
data.tar.gz: 5d06d43f7367afc011bb3dce3c33c95cbbc7205238e08bdf2f494426e2c6fc90088c6854a55f8a49712bec79a54ed01aacb7358d42e1e0bb50ec5fc68d9b8126
|
data/README.md
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
# TWB - The Tableau Workbook Ruby gem.
|
|
2
2
|
|
|
3
|
-
This gem models Tableau workbooks and their major components. It's the result
|
|
4
|
-
and manage Tableau workbooks
|
|
3
|
+
This gem models Tableau workbooks and their major components. It's the result of years of writing similar code to access, interpret,
|
|
4
|
+
and manage Tableau workbooks.
|
|
5
5
|
|
|
6
6
|
## Philosophy
|
|
7
7
|
|
data/lib/twb.rb
CHANGED
|
@@ -16,7 +16,6 @@
|
|
|
16
16
|
require_relative 'twb/dashboard'
|
|
17
17
|
require_relative 'twb/datasource'
|
|
18
18
|
require_relative 'twb/docdashboard'
|
|
19
|
-
require_relative 'twb/fieldcalculation.rb'
|
|
20
19
|
require_relative 'twb/localfield'
|
|
21
20
|
require_relative 'twb/metadatafield'
|
|
22
21
|
require_relative 'twb/storyboard'
|
|
@@ -24,9 +23,11 @@ require_relative 'twb/window'
|
|
|
24
23
|
require_relative 'twb/workbook'
|
|
25
24
|
require_relative 'twb/worksheet'
|
|
26
25
|
require_relative 'twb/action'
|
|
26
|
+
require_relative 'twb/columnfield.rb'
|
|
27
|
+
require_relative 'twb/calculatedfield'
|
|
27
28
|
require_relative 'twb/fieldcalculation'
|
|
28
|
-
require_relative 'twb/docdashboardimagevert
|
|
29
|
-
require_relative 'twb/docdashboardwebvert
|
|
29
|
+
require_relative 'twb/docdashboardimagevert'
|
|
30
|
+
require_relative 'twb/docdashboardwebvert'
|
|
30
31
|
require_relative 'twb/util/twbDashSheetDataDotBuilder'
|
|
31
32
|
require_relative 'twb/util/dotFileRenderer'
|
|
32
33
|
require_relative 'twb/util/htmllistcollapsible'
|
|
@@ -34,11 +35,14 @@ require_relative 'twb/util/xraydashboards'
|
|
|
34
35
|
require_relative 'twb/util/graphnode'
|
|
35
36
|
require_relative 'twb/util/graphedge'
|
|
36
37
|
require_relative 'twb/util/graphedges'
|
|
37
|
-
require_relative 'twb/analysis/calculatedfieldsanalyzer'
|
|
38
|
-
|
|
38
|
+
require_relative 'twb/analysis/calculatedfields/calculatedfieldsanalyzer'
|
|
39
|
+
require_relative 'twb/analysis/calculatedfields/markdownemitter'
|
|
40
|
+
require_relative 'twb/analysis/calculatedfields/csvemitter'
|
|
41
|
+
require_relative 'twb/analysis/datasources/DataSourceTableFieldsCSVEmitter'
|
|
42
|
+
require_relative 'twb/analysis/Sheets/WorksheetDataStructureCSVEmitter.rb'
|
|
39
43
|
|
|
40
44
|
# Represents Tableau Workbooks and their contents.
|
|
41
45
|
#
|
|
42
46
|
module Twb
|
|
43
|
-
VERSION = '1.
|
|
47
|
+
VERSION = '1.9.1'
|
|
44
48
|
end
|
|
@@ -0,0 +1,154 @@
|
|
|
1
|
+
# calculatedfieldsanalyzer.rb - this Ruby script Copyright 2017 Christopher Gerrard
|
|
2
|
+
#
|
|
3
|
+
# This program is free software: you can redistribute it and/or modify
|
|
4
|
+
# it under the terms of the GNU General Public License as published by
|
|
5
|
+
# the Free Software Foundation, either version 3 of the License, or
|
|
6
|
+
# (at your option) any later version.
|
|
7
|
+
#
|
|
8
|
+
# This program is distributed in the hope that it will be useful,
|
|
9
|
+
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
10
|
+
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
11
|
+
# GNU General Public License for more details.
|
|
12
|
+
#
|
|
13
|
+
# You should have received a copy of the GNU General Public License
|
|
14
|
+
# along with this program. If not, see <http://www.gnu.org/licenses/>.
|
|
15
|
+
|
|
16
|
+
require 'twb'
|
|
17
|
+
|
|
18
|
+
module Twb
|
|
19
|
+
module Analysis
|
|
20
|
+
module CalculatedFields
|
|
21
|
+
|
|
22
|
+
class CSVEmitter
|
|
23
|
+
|
|
24
|
+
attr_reader :calcFieldsType, :calcFieldsHeader, :calcFieldsFile, :calcFieldsRecords, :calcFieldsCount
|
|
25
|
+
attr_reader :refFieldsType, :refFieldsHeader, :refFieldsFile, :refFieldsRecords, :refFieldsCount
|
|
26
|
+
attr_reader :dictionary, :csvFiles
|
|
27
|
+
|
|
28
|
+
def initialize
|
|
29
|
+
@calcFieldsContents = 'Calculated Fields with Formulas'
|
|
30
|
+
@calcFieldsType = 'CalculatedFieldsWithFormulas'
|
|
31
|
+
@calcFieldsHeader = ['Record #',
|
|
32
|
+
'Workbook', 'Workbook Dir',
|
|
33
|
+
'Data Source Name (tech)', 'Data Source Caption', 'Data Source Name',
|
|
34
|
+
'Field Name (tech)', 'Field Caption', 'Field Name',
|
|
35
|
+
'Data Source + Field Name (tech)',
|
|
36
|
+
'Formula (flat)',
|
|
37
|
+
'Formula Length',
|
|
38
|
+
'Formula Table Calc?',
|
|
39
|
+
'Formula LOD?',
|
|
40
|
+
'Formula LOD Code Position'
|
|
41
|
+
]
|
|
42
|
+
|
|
43
|
+
@refFieldsContents = 'Fields referenced in Calculated Fields Formulae'
|
|
44
|
+
@refFieldsType = 'CalculatedFieldsReferencedFields'
|
|
45
|
+
@refFieldsHeader = ['Record #',
|
|
46
|
+
'Workbook', 'Workbook Dir',
|
|
47
|
+
'Data Source Name (tech)', 'Data Source Caption', 'Data Source Name',
|
|
48
|
+
'Field Name (tech)', 'Field Caption', 'Field Name',
|
|
49
|
+
'Referenced Field',
|
|
50
|
+
'Referenced DataSource',
|
|
51
|
+
'Referenced DataSource Location'
|
|
52
|
+
]
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
@dictionary = [
|
|
56
|
+
{ :contents => @calcFieldsContents,
|
|
57
|
+
:fileType => @calcFieldsType,
|
|
58
|
+
:fileHeader => @calcFieldsHeader
|
|
59
|
+
},
|
|
60
|
+
{ :contents => @refFieldsContents,
|
|
61
|
+
:fileType => @calcFieldsType,
|
|
62
|
+
:fileHeader => @calcFieldsHeader
|
|
63
|
+
}
|
|
64
|
+
]
|
|
65
|
+
end
|
|
66
|
+
|
|
67
|
+
# def dictionary
|
|
68
|
+
# @dictionary
|
|
69
|
+
# end
|
|
70
|
+
|
|
71
|
+
# def calcFieldsType
|
|
72
|
+
# @calcFieldsType
|
|
73
|
+
# end
|
|
74
|
+
|
|
75
|
+
# def calcFieldsHeader
|
|
76
|
+
# @calcFieldsHeader
|
|
77
|
+
# end
|
|
78
|
+
|
|
79
|
+
# def refFieldsType
|
|
80
|
+
# @refFieldsType
|
|
81
|
+
# end
|
|
82
|
+
|
|
83
|
+
# def calcFieldsHeader
|
|
84
|
+
# @refFieldsHeader
|
|
85
|
+
# end
|
|
86
|
+
|
|
87
|
+
def processTwb twbFileName
|
|
88
|
+
twb = File.basename(twbFileName)
|
|
89
|
+
@twb = Twb::Workbook.new twb
|
|
90
|
+
# --
|
|
91
|
+
@calcFieldsFile = "#{twb}.#{@calcFieldsType}.csv"
|
|
92
|
+
@csvCalcFile = CSV.open(@calcFieldsFile,'w')
|
|
93
|
+
@csvCalcFile << @calcFieldsHeader
|
|
94
|
+
@calcFieldsCount = 0
|
|
95
|
+
@calcFieldsRecords = []
|
|
96
|
+
# --
|
|
97
|
+
@refFieldsFile = "#{twb}.#{@refFieldsType}.csv" # twb + '.CalcReferenceFields.csv'
|
|
98
|
+
@csvRefFile = CSV.open(@refFieldsFile,'w')
|
|
99
|
+
@csvRefFile << @refFieldsHeader
|
|
100
|
+
@refFieldsCount = 0
|
|
101
|
+
@refFieldsRecords = []
|
|
102
|
+
# --
|
|
103
|
+
@csvFiles = [
|
|
104
|
+
{ :contents => @calcFieldsContents,
|
|
105
|
+
:name => @calcFieldsFile,
|
|
106
|
+
:records => @calcFieldsRecords
|
|
107
|
+
},
|
|
108
|
+
{ :contents => @refFieldsContents,
|
|
109
|
+
:name => @refFieldsFile,
|
|
110
|
+
:records => @refFieldsRecords
|
|
111
|
+
}
|
|
112
|
+
]
|
|
113
|
+
# --
|
|
114
|
+
dss = @twb.datasources
|
|
115
|
+
dss.each do |ds|
|
|
116
|
+
calcFields = ds.calculatedFieldsMap.sort_by { |fldName,calc| fldName }
|
|
117
|
+
calcFields.each do |fldname, field|
|
|
118
|
+
formRecord = [ @calcFieldsCount += 1,
|
|
119
|
+
@twb.name, @twb.dir,
|
|
120
|
+
ds.name, ds.caption, ds.uiname,
|
|
121
|
+
field.name, field.caption, field.uiname,
|
|
122
|
+
"#{ds.name}.#{field.name}",
|
|
123
|
+
field.calculation.formulaFlatResolved,
|
|
124
|
+
field.calculation.formula.length,
|
|
125
|
+
field.calculation.is_tableCalc,
|
|
126
|
+
field.calculation.is_lod,
|
|
127
|
+
field.calculation.lodCodePos
|
|
128
|
+
]
|
|
129
|
+
@calcFieldsRecords.push formRecord
|
|
130
|
+
@csvCalcFile << formRecord
|
|
131
|
+
#--
|
|
132
|
+
field.calculation.calcFields.each do |cf|
|
|
133
|
+
refRecord = [ @refFieldsCount += 1,
|
|
134
|
+
@twb.name, @twb.dir,
|
|
135
|
+
ds.name, ds.caption, ds.uiname,
|
|
136
|
+
field.name, field.caption, field.uiname,
|
|
137
|
+
cf.uiName,
|
|
138
|
+
cf.dataSource,
|
|
139
|
+
cf.dataSourceRef,
|
|
140
|
+
]
|
|
141
|
+
@refFieldsRecords.push refRecord
|
|
142
|
+
@csvRefFile << refRecord
|
|
143
|
+
end
|
|
144
|
+
end
|
|
145
|
+
end
|
|
146
|
+
@csvCalcFile.close unless @csvCalcFile.nil?
|
|
147
|
+
@csvRefFile.close unless @csvRefFile.nil?
|
|
148
|
+
end
|
|
149
|
+
|
|
150
|
+
end # class CSVEmitter
|
|
151
|
+
|
|
152
|
+
end # nodule CalculatedFields
|
|
153
|
+
end # module Analysis
|
|
154
|
+
end # module Twb
|
|
@@ -0,0 +1,527 @@
|
|
|
1
|
+
# calculatedfieldsanalyzer.rb - this Ruby script Copyright 2017 Christopher Gerrard
|
|
2
|
+
#
|
|
3
|
+
# This program is free software: you can redistribute it and/or modify
|
|
4
|
+
# it under the terms of the GNU General Public License as published by
|
|
5
|
+
# the Free Software Foundation, either version 3 of the License, or
|
|
6
|
+
# (at your option) any later version.
|
|
7
|
+
#
|
|
8
|
+
# This program is distributed in the hope that it will be useful,
|
|
9
|
+
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
10
|
+
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
11
|
+
# GNU General Public License for more details.
|
|
12
|
+
#
|
|
13
|
+
# You should have received a copy of the GNU General Public License
|
|
14
|
+
# along with this program. If not, see <http://www.gnu.org/licenses/>.
|
|
15
|
+
|
|
16
|
+
require 'nokogiri'
|
|
17
|
+
require 'twb'
|
|
18
|
+
require 'set'
|
|
19
|
+
require 'csv'
|
|
20
|
+
require 'logger'
|
|
21
|
+
|
|
22
|
+
module Twb
|
|
23
|
+
module Analysis
|
|
24
|
+
|
|
25
|
+
class CalculatedFieldsAnalyzer
|
|
26
|
+
|
|
27
|
+
attr_reader :calculatedFieldsCount, :formulaFieldsCount, :dataFiles
|
|
28
|
+
|
|
29
|
+
@@ttlogfile = 'CalculatedFieldsAnalyzer.ttlog'
|
|
30
|
+
@@gvDotLocation = 'C:\\tech\\graphviz\\Graphviz2.38\\bin\\dot.exe'
|
|
31
|
+
@@processName = '.CalculatedFields'
|
|
32
|
+
|
|
33
|
+
@@calcFieldsCSVFileName = 'TwbCalculatedFields.csv'
|
|
34
|
+
@@calcFieldsCSVFileHeader = ['Record #',
|
|
35
|
+
'Workbook', 'Workbook Dir',
|
|
36
|
+
'Data Source', 'Data Source Caption', 'Data Source Name (tech)',
|
|
37
|
+
'Field Name', 'Field Caption', 'Field Name (tech)',
|
|
38
|
+
'Data Source + Field Name (tech)',
|
|
39
|
+
'Data Type', 'Role', 'Type',
|
|
40
|
+
'Class',
|
|
41
|
+
'Scope Isolation',
|
|
42
|
+
'Formula Length',
|
|
43
|
+
'Formula Code',
|
|
44
|
+
'Formula',
|
|
45
|
+
'Formula Comments',
|
|
46
|
+
'Formula LOD?'
|
|
47
|
+
]
|
|
48
|
+
|
|
49
|
+
@@formFieldsCSVFileName = 'TwbFormulaFields.csv'
|
|
50
|
+
@@formFieldsCSVFileHeader = ['Rec #',
|
|
51
|
+
'Workbook', 'Workbook Dir',
|
|
52
|
+
'Data Source',
|
|
53
|
+
'Field - Calculated',
|
|
54
|
+
'Data Source - Formula (tech)',
|
|
55
|
+
'Data Source - Formula',
|
|
56
|
+
'Field - Formula (tech)',
|
|
57
|
+
'Field - Formula',
|
|
58
|
+
'Data Source + Field - Calculated',
|
|
59
|
+
'Table'
|
|
60
|
+
]
|
|
61
|
+
|
|
62
|
+
@techUINames = {}
|
|
63
|
+
@fieldTables = {}
|
|
64
|
+
|
|
65
|
+
|
|
66
|
+
@@dotHeader = <<DOTHEADER
|
|
67
|
+
digraph g {
|
|
68
|
+
graph [rankdir="LR" splines=line];
|
|
69
|
+
node [shape="box" width="2"];
|
|
70
|
+
|
|
71
|
+
DOTHEADER
|
|
72
|
+
|
|
73
|
+
def initialize
|
|
74
|
+
@csvCalculatedFields = []
|
|
75
|
+
@csvFormulaFields = []
|
|
76
|
+
# @testFile = File.open('testCSV.csv','w')
|
|
77
|
+
# @testFile.puts @@calcFieldsCSVFileHeader.inspect
|
|
78
|
+
#-- Logging setup --
|
|
79
|
+
@logger = Logger.new(@@ttlogfile)
|
|
80
|
+
@logger.level = Logger::DEBUG
|
|
81
|
+
#-- Counters setup --
|
|
82
|
+
@twbCount = 0
|
|
83
|
+
@calculatedFieldsCount = 0
|
|
84
|
+
@formulaFieldsCount = 0
|
|
85
|
+
# --
|
|
86
|
+
@referencedFields = SortedSet.new
|
|
87
|
+
# --
|
|
88
|
+
@dataFiles = {'TwbCalculatedFields.csv' => 'Calculated Fields & their Formulas', 'TwbFormulaFields.csv' => 'Fields referenced in Formulas'}
|
|
89
|
+
# --
|
|
90
|
+
@localEmit = false
|
|
91
|
+
emit "\n\nLogging activity to: #{File.basename(@@ttlogfile)}"
|
|
92
|
+
@imageFiles = []
|
|
93
|
+
end
|
|
94
|
+
|
|
95
|
+
def processTWB twbWithDir
|
|
96
|
+
twb = File.basename(twbWithDir)
|
|
97
|
+
@twb = Twb::Workbook.new twbWithDir
|
|
98
|
+
emit "- Workbook: #{twbWithDir}"
|
|
99
|
+
emit " version: #{@twb.version}"
|
|
100
|
+
return if twbWithDir.end_with? == "Tableau Calculated Fields Analyses.twb"
|
|
101
|
+
twbDir = File.dirname(File.expand_path(twbWithDir))
|
|
102
|
+
edges = Set.new
|
|
103
|
+
# -- processing
|
|
104
|
+
dss = @twb.datasources
|
|
105
|
+
puts " # data sources: #{dss.length}"
|
|
106
|
+
twbRootFields = Set.new
|
|
107
|
+
@twbFields = {}
|
|
108
|
+
dss.each do |ds|
|
|
109
|
+
puts "\t\t - #{ds.uiname} \t\t #{ds.calculatedFields.length}"
|
|
110
|
+
next if ds.Parameters? # don't process the Parameters data source
|
|
111
|
+
ds.calculatedFields.each do |calcField|
|
|
112
|
+
emit "HANDLING CALCULATED FIELD:: #{calcField}"
|
|
113
|
+
emit " :: #{calcField.calculation.formula}"
|
|
114
|
+
emit " :: #{calcField.calculation.formulaResolved}"
|
|
115
|
+
dsTechName = ds.name
|
|
116
|
+
dsCaption = ds.caption
|
|
117
|
+
dsName = ds.uiname
|
|
118
|
+
dsID = dsTechName + ':::' + dsName
|
|
119
|
+
emit "\n\n "
|
|
120
|
+
emit "======================================================"
|
|
121
|
+
emit "======================================================"
|
|
122
|
+
emit "======= DATA SOURCE: #{ds.uiname} ====== "
|
|
123
|
+
emit "======================================================"
|
|
124
|
+
emit "======================================================\n\n "
|
|
125
|
+
dsGraphNode = Twb::Util::Graphnode.new(name: dsName, id: dsID, type: :TwbDataConnection, properties: {workbook: twbWithDir})
|
|
126
|
+
emit "\t dsgnode: #{dsGraphNode}"
|
|
127
|
+
fieldUINames = ds.fieldUINames
|
|
128
|
+
emit "ds.calculatedFields :: nil? #{ds.calculatedFields.nil?}"
|
|
129
|
+
end
|
|
130
|
+
processDataSource ds
|
|
131
|
+
end
|
|
132
|
+
end
|
|
133
|
+
|
|
134
|
+
def processDataSource ds
|
|
135
|
+
emit "Datasource: '#{ds.uiname}' -> #{ds.Parameters?}"
|
|
136
|
+
dsFields = {}
|
|
137
|
+
@twbFields[ds.uiname] = dsFields
|
|
138
|
+
# next if ds.Parameters? # don't process the Parameters data source
|
|
139
|
+
# it requires special handling, has different XML structure
|
|
140
|
+
#-- For tracking unreferenced (root) calculated fields = calculatedFields - referencedFields
|
|
141
|
+
calculatedFields = SortedSet.new
|
|
142
|
+
referencedFields = SortedSet.new
|
|
143
|
+
#--
|
|
144
|
+
dsTechName = ds.name
|
|
145
|
+
dsCaption = ds.caption
|
|
146
|
+
dsName = ds.uiname
|
|
147
|
+
dsID = dsTechName + ':::' + dsName
|
|
148
|
+
emit "\n\n "
|
|
149
|
+
emit "======================================================"
|
|
150
|
+
emit "======================================================"
|
|
151
|
+
emit "======= DATA SOURCE: #{ds.uiname} ====== "
|
|
152
|
+
emit "======================================================"
|
|
153
|
+
emit "======================================================\n\n "
|
|
154
|
+
dsGraphNode = Twb::Util::Graphnode.new(name: dsName, id: dsID, type: :TwbDataConnection, properties: {workbook: twbWithDir})
|
|
155
|
+
emit "\t dsgnode: #{dsGraphNode}"
|
|
156
|
+
fieldUINames = ds.fieldUINames
|
|
157
|
+
emit "ds.calculatedFields :: nil? #{ds.calculatedFields.nil?}"
|
|
158
|
+
ds.calculatedFields.each do |calcField|
|
|
159
|
+
emit "HANDLING CALCULATED FIELD:: #{calcField}"
|
|
160
|
+
emit '--'
|
|
161
|
+
fldCaption, = calcField.caption
|
|
162
|
+
fldTechName = calcField.name
|
|
163
|
+
fldName = calcField.uiname
|
|
164
|
+
dataType = calcField.datatype
|
|
165
|
+
role = calcField.role
|
|
166
|
+
type = calcField.type
|
|
167
|
+
fieldID = fldTechName+'::'+dsName
|
|
168
|
+
calculatedFields.add fieldID
|
|
169
|
+
#--
|
|
170
|
+
dsFields[fldName] = calcField
|
|
171
|
+
srcGraphNode = Twb::Util::Graphnode.new(name: fldName, id: fieldID, type: :CalculatedField, properties: {:DataSource => dsName})
|
|
172
|
+
dsFieldEdge = Twb::Util::Graphedge.new(from: dsGraphNode, to: srcGraphNode, relationship: 'contains')
|
|
173
|
+
#--
|
|
174
|
+
emit "\t srfnode: #{srcGraphNode} "
|
|
175
|
+
emit "\t dsfedge: #{dsFieldEdge} "
|
|
176
|
+
edges.add dsFieldEdge
|
|
177
|
+
calculation = calcField.calculation
|
|
178
|
+
emit "calculation: f? %8s -> %s " % [calculation.has_formula, calculation.formula ]
|
|
179
|
+
if calculation.has_formula
|
|
180
|
+
formulaFlat = calculation.formulaFlat
|
|
181
|
+
uiFormula = formulaFlat.gsub(' XX ',' ')
|
|
182
|
+
formulaLOD = formulaFlat.upcase =~ /^[ ]*{[ ]*(INCLUDE|FIXED|EXCLUDE)/
|
|
183
|
+
resolvedFields = calculation.resolvedFields
|
|
184
|
+
resolvedFields.each do |rf|
|
|
185
|
+
emit "\tRESOLVED FLD: #{rf.inspect}"
|
|
186
|
+
calcFieldName = rf[:field]
|
|
187
|
+
if rf[:source].nil?
|
|
188
|
+
calcFieldRef = "[%s]" % [ calcFieldName ]
|
|
189
|
+
dispFieldRef = "[%s]" % [ ds.fieldUIName(calcFieldName) ]
|
|
190
|
+
else
|
|
191
|
+
remoteDS = @twb.datasource(rf[:source])
|
|
192
|
+
if remoteDS.nil?
|
|
193
|
+
calcFieldRef = "[DS_NOT_FOUND].[%s]" % [ calcFieldName ]
|
|
194
|
+
dispFieldRef = calcFieldRef
|
|
195
|
+
else
|
|
196
|
+
remoteDSName = remoteDS.uiname
|
|
197
|
+
remoteDSFld = remoteDS.fieldUIName(calcFieldName)
|
|
198
|
+
calcFieldRef = "[%s].[%s]" % [ rf[:source], calcFieldName ]
|
|
199
|
+
dispFieldRef = "[%s].[%s]" % [ remoteDSName, remoteDSFld ]
|
|
200
|
+
end # remoteDS.nil?
|
|
201
|
+
end # rf[:source].nil?
|
|
202
|
+
emit "\tcalcFieldRef: #{calcFieldRef}"
|
|
203
|
+
emit "\tdispFieldRef: #{dispFieldRef}"
|
|
204
|
+
uiFormula = uiFormula.gsub(calcFieldRef, dispFieldRef)
|
|
205
|
+
end # resolvedFields.each
|
|
206
|
+
@csvCalculatedFields.push [
|
|
207
|
+
@calculatedFieldsCount += 1,
|
|
208
|
+
twb,
|
|
209
|
+
twbDir,
|
|
210
|
+
dsName,
|
|
211
|
+
dsCaption,
|
|
212
|
+
dsTechName,
|
|
213
|
+
fldName,
|
|
214
|
+
fldCaption,
|
|
215
|
+
fldTechName,
|
|
216
|
+
dsTechName + '::' + fldTechName,
|
|
217
|
+
dataType,
|
|
218
|
+
role,
|
|
219
|
+
type,
|
|
220
|
+
calculation.class,
|
|
221
|
+
calculation.scopeIsolation,
|
|
222
|
+
calculation.formulaFlat.length,
|
|
223
|
+
calculation.formulaFlat,
|
|
224
|
+
uiFormula[0...200],
|
|
225
|
+
calculation.comments,
|
|
226
|
+
!formulaLOD.nil?
|
|
227
|
+
]
|
|
228
|
+
resolvedFields.each do |rf|
|
|
229
|
+
emit "\t\t res field : #{rf[:field]} "
|
|
230
|
+
emit "\t\t res source: #{rf[:source]}"
|
|
231
|
+
calcFieldName = rf[:field]
|
|
232
|
+
calcDataSource = rf[:source]
|
|
233
|
+
localDataSource = rf[:source].nil? # if there isn't a rf[:source] value
|
|
234
|
+
# the field is from this data source
|
|
235
|
+
# else the field is from an alien data source (in the same workbook)
|
|
236
|
+
refDataSource = localDataSource ? ds : @twb.datasource(calcDataSource) # data source may not be in Workbook
|
|
237
|
+
refDataSourceName = !refDataSource.nil? ? refDataSource.uiname : calcDataSource + '\n***DATA CONNECTION NOT IN WORKBOOK***'
|
|
238
|
+
if !refDataSource.nil?
|
|
239
|
+
dispFieldName = refDataSource.fieldUIName(calcFieldName)
|
|
240
|
+
calcFieldTable = refDataSource.fieldTable(calcFieldName)
|
|
241
|
+
else
|
|
242
|
+
dispFieldName = calcFieldName
|
|
243
|
+
calcFieldTable = calcFieldName
|
|
244
|
+
end
|
|
245
|
+
emit "\t\t calc field : #{dispFieldName} nil?<#{dispFieldName.nil?}>"
|
|
246
|
+
emit "\t\t data source: #{refDataSourceName}"
|
|
247
|
+
emit "\t\t table: #{calcFieldTable} nil?<#{calcFieldTable.nil?}>"
|
|
248
|
+
properties = {'DataSource' => dsName, 'DataSourceReference' => 'local'}
|
|
249
|
+
if dispFieldName.nil?
|
|
250
|
+
dispFieldName = "<#{calcFieldName}>::<#{calcDataSource}> UNDEFINED"
|
|
251
|
+
properties['status'] = 'UNDEFINED'
|
|
252
|
+
end
|
|
253
|
+
calcFieldID = "#{calcFieldName}::#{refDataSourceName}"
|
|
254
|
+
if !localDataSource
|
|
255
|
+
calcFieldID = "#{calcFieldName}:LDS:#{ds.uiname}:RDS:#{refDataSourceName}"
|
|
256
|
+
properties['DataSourceReference'] = 'remote'
|
|
257
|
+
end
|
|
258
|
+
calcFieldTable = ds.fieldTable(calcFieldName)
|
|
259
|
+
calcFieldType = calcFieldTable.nil? ? :CalculatedField : :DatabaseField
|
|
260
|
+
calcFieldNode = Twb::Util::Graphnode.new(name: dispFieldName, id: calcFieldID, type: calcFieldType, properties: properties)
|
|
261
|
+
fieldFieldEdge = Twb::Util::Graphedge.new(from: srcGraphNode, to: calcFieldNode, relationship: 'references')
|
|
262
|
+
edges.add fieldFieldEdge
|
|
263
|
+
referencedFields.add calcFieldID
|
|
264
|
+
emit "\t\t calcFieldNode: #{calcFieldNode}"
|
|
265
|
+
emit "\t\t graphEdge: #{fieldFieldEdge}"
|
|
266
|
+
fldToDsNode = calcFieldNode
|
|
267
|
+
if !calcFieldTable.nil?
|
|
268
|
+
tableID = calcFieldTable + ':::' + ds.uiname
|
|
269
|
+
tableName = "-[#{calcFieldTable}]-"
|
|
270
|
+
tableNode = Twb::Util::Graphnode.new(name: tableName, id: tableID, type: :DBTable, properties: properties)
|
|
271
|
+
fieldFieldEdge = Twb::Util::Graphedge.new(from: calcFieldNode, to: tableNode, relationship: 'is a field in')
|
|
272
|
+
edges.add fieldFieldEdge
|
|
273
|
+
fldToDsNode = tableNode
|
|
274
|
+
end
|
|
275
|
+
if !localDataSource
|
|
276
|
+
alienDSNode = Twb::Util::Graphnode.new( name: '==>' + refDataSourceName,
|
|
277
|
+
id: "#{ds.uiname}::::=>#{refDataSourceName}",
|
|
278
|
+
type: :DBTable,
|
|
279
|
+
properties: {'Home Source' => dsName, 'Remote Source' => refDataSourceName}
|
|
280
|
+
)
|
|
281
|
+
fieldFieldEdge = Twb::Util::Graphedge.new(from: fldToDsNode, to: alienDSNode, relationship: 'In Remote Data Source')
|
|
282
|
+
edges.add fieldFieldEdge
|
|
283
|
+
end
|
|
284
|
+
|
|
285
|
+
# @csvFormulaFields.push [ @formulaFieldsCount+=1,
|
|
286
|
+
|
|
287
|
+
|
|
288
|
+
end # resolvedFields.each do
|
|
289
|
+
end # if calculation.has_formula
|
|
290
|
+
end # ds.calculatedFields.each
|
|
291
|
+
|
|
292
|
+
dsRootFields = calculatedFields - referencedFields
|
|
293
|
+
@referencedFields.merge referencedFields
|
|
294
|
+
#--
|
|
295
|
+
emit "--\nCalculated Fields\n-----------------"
|
|
296
|
+
calculatedFields.each { |f| emit f }
|
|
297
|
+
emit "--\nReferenced Fields\n-----------------"
|
|
298
|
+
referencedFields.each { |f| emit f }
|
|
299
|
+
emit "--\nDS Root Fields\n-----------------"
|
|
300
|
+
dsRootFields.each { |f| emit f }
|
|
301
|
+
emit "--"
|
|
302
|
+
# --
|
|
303
|
+
twbRootFields.merge dsRootFields
|
|
304
|
+
@twbCount += 1
|
|
305
|
+
mapTwb twb, edges, twbRootFields
|
|
306
|
+
graphEdges twb, edges
|
|
307
|
+
emit "#######################"
|
|
308
|
+
#--
|
|
309
|
+
csvCF = File.open(@@calcFieldsCSVFileName, 'w')
|
|
310
|
+
csvCF.puts @@calcFieldsCSVFileHeader.to_csv
|
|
311
|
+
@csvCalculatedFields.each do |r|
|
|
312
|
+
csvCF.puts r.to_csv
|
|
313
|
+
end
|
|
314
|
+
csvCF.close
|
|
315
|
+
#--
|
|
316
|
+
csvFF = File.open(@@formFieldsCSVFileName, 'w')
|
|
317
|
+
csvFF.puts @@formFieldsCSVFileHeader.to_csv
|
|
318
|
+
@csvFormulaFields.each do |r|
|
|
319
|
+
csvFF.puts r.to_csv
|
|
320
|
+
end
|
|
321
|
+
csvFF.close
|
|
322
|
+
#--
|
|
323
|
+
return @imageFiles
|
|
324
|
+
end # def processDataSource
|
|
325
|
+
|
|
326
|
+
def emitCalcfield calcField
|
|
327
|
+
emit "\t FIELD cap :: #{calcField.caption} "
|
|
328
|
+
emit "\t tname:: #{calcField.name}"
|
|
329
|
+
emit "\t uiname:: #{calcField.uiname}"
|
|
330
|
+
emit "\t formula:: #{calculation.formulaFlat}"
|
|
331
|
+
end
|
|
332
|
+
|
|
333
|
+
def mapTwb twb, edges, rootFields
|
|
334
|
+
dotFile = initDot twb
|
|
335
|
+
dotFileName = File.basename dotFile
|
|
336
|
+
dotFile.puts "\n // subgraph cluster_1 {"
|
|
337
|
+
dotFile.puts " // color= grey;"
|
|
338
|
+
dotFile.puts ""
|
|
339
|
+
edgesAsStrings = SortedSet.new
|
|
340
|
+
# this two step process coalesces the edges into a unique set, avoiding duplicating the dot
|
|
341
|
+
# file entries, and can be shrunk when graph edges expose the bits necessary for management by Set
|
|
342
|
+
emit "\n========================\nLoading Edges\n========================\n From DC? Referenced? Edge \n %s %s %s" % ['--------', '-----------', '-'*45]
|
|
343
|
+
edges.each do |e|
|
|
344
|
+
# don't want to emit edge which is from a Data Connection to a
|
|
345
|
+
# Calculated Field which is also referenced by another calculated field
|
|
346
|
+
isFromDC = e.from.type == :TwbDataConnection
|
|
347
|
+
isRefField = @referencedFields.include?(e.to.id)
|
|
348
|
+
edgesAsStrings.add(e.dot) unless isFromDC && isRefField
|
|
349
|
+
end
|
|
350
|
+
emit "------------------------\n "
|
|
351
|
+
edgesAsStrings.each do |es|
|
|
352
|
+
dotFile.puts " #{es}"
|
|
353
|
+
emit " #{es}"
|
|
354
|
+
end
|
|
355
|
+
emit "========================\n "
|
|
356
|
+
dotFile.puts ""
|
|
357
|
+
dotFile.puts " // }"
|
|
358
|
+
dotFile.puts "\n\n // 4--------------------------------------------------------------------"
|
|
359
|
+
# "table::JIRA_HARVEST_Correspondence__c::Jira" [label="JIRA_HARVEST_Correspondence__c"]
|
|
360
|
+
nodes = SortedSet.new
|
|
361
|
+
edges.each do |e|
|
|
362
|
+
nodes.add e.from.dotLabel
|
|
363
|
+
nodes.add e.to.dotLabel
|
|
364
|
+
end
|
|
365
|
+
nodes.each do |n|
|
|
366
|
+
dotFile.puts n
|
|
367
|
+
end
|
|
368
|
+
dotFile.puts "\n\n // 5--------------------------------------------------------------------"
|
|
369
|
+
emitTypes( edges, dotFile )
|
|
370
|
+
rankRootFields( dotFile, rootFields )
|
|
371
|
+
closeDot( dotFile, twb )
|
|
372
|
+
# renderPng(twb.name,dotFileName)
|
|
373
|
+
# renderPdf(twb.name,dotFileName)
|
|
374
|
+
renderDot(twb,dotFileName,'pdf')
|
|
375
|
+
renderDot(twb,dotFileName,'png')
|
|
376
|
+
renderDot(twb,dotFileName,'svg')
|
|
377
|
+
emitEdges edges
|
|
378
|
+
end
|
|
379
|
+
|
|
380
|
+
|
|
381
|
+
def graphEdges twb, edges
|
|
382
|
+
graphFile = File.new(twb + '.cypher', 'w')
|
|
383
|
+
# graphFile.puts "OKEY DOKE, graphing away"
|
|
384
|
+
cypherCode = Set.new
|
|
385
|
+
edges.each do |edge|
|
|
386
|
+
cypherCode.add edge.from.cypherCreate
|
|
387
|
+
cypherCode.add edge.to.cypherCreate
|
|
388
|
+
cypherCode.add edge.cypherCreate
|
|
389
|
+
end
|
|
390
|
+
cypherCode.each do |cc|
|
|
391
|
+
graphFile.puts cc
|
|
392
|
+
end
|
|
393
|
+
graphFile.puts "\nreturn *"
|
|
394
|
+
graphFile.close unless graphFile.nil?
|
|
395
|
+
@imageFiles << File.basename(graphFile)
|
|
396
|
+
end
|
|
397
|
+
|
|
398
|
+
def emitEdges edges
|
|
399
|
+
emit " %-15s %s" % ['type', 'Edge']
|
|
400
|
+
emit " %-15s %s" % ['-'*15, '-'*35]
|
|
401
|
+
edges.each do |edge|
|
|
402
|
+
emit " %-15s %s" % [edge.from.type, edge.from]
|
|
403
|
+
emit " %-15s %s" % [edge.to.type, edge.to]
|
|
404
|
+
emit "\n "
|
|
405
|
+
end
|
|
406
|
+
end
|
|
407
|
+
|
|
408
|
+
def emitTypes edges, dotFile
|
|
409
|
+
typedNodes = {}
|
|
410
|
+
dotFile.puts "\n\n // 2--------------------------------------------------------------------"
|
|
411
|
+
edges.each do |edge|
|
|
412
|
+
emit " EDGE :: #{edge}"
|
|
413
|
+
loadNodeType typedNodes, edge.from
|
|
414
|
+
loadNodeType typedNodes, edge.to
|
|
415
|
+
end
|
|
416
|
+
typedNodes.each do |type, nodes|
|
|
417
|
+
emit "+++++++++ typedNodes of '#{type}'' "
|
|
418
|
+
nodes.each do |node|
|
|
419
|
+
emit " -n- #{node}"
|
|
420
|
+
end
|
|
421
|
+
rankSame(dotFile, type, nodes) unless type == :CalculatedField
|
|
422
|
+
end
|
|
423
|
+
# labelTypes dotFile, edges
|
|
424
|
+
end
|
|
425
|
+
|
|
426
|
+
def loadNodeType set, node
|
|
427
|
+
type = node.type
|
|
428
|
+
set[type] = Set.new unless set.include? type
|
|
429
|
+
set[type].add node
|
|
430
|
+
end
|
|
431
|
+
|
|
432
|
+
def rankSame dotFile, type, nodes
|
|
433
|
+
dotFile.puts "\n // '#{type}' --------------------------------------------------------------------"
|
|
434
|
+
dotFile.puts "\n {rank=same "
|
|
435
|
+
# dotFile.puts " \"#{type}\" [shape=\"box3d\" style=\"filled\" ]" unless ''.eql? type # [shape=\"box3d\" style=\"filled\" ]\"" unless label.equal? ''
|
|
436
|
+
nodes.each do |node|
|
|
437
|
+
dotFile.puts " \"#{node.id}\""
|
|
438
|
+
end
|
|
439
|
+
dotFile.puts " }"
|
|
440
|
+
end
|
|
441
|
+
|
|
442
|
+
def rankRootFields dotFile, dsRootFields
|
|
443
|
+
dotFile.puts "\n // Unreferenced (root) Calculated Fields -----------------------------------------"
|
|
444
|
+
dotFile.puts "\n {rank=same "
|
|
445
|
+
dsRootFields.each do |rf|
|
|
446
|
+
dotFile.puts " \"#{rf}\""
|
|
447
|
+
end
|
|
448
|
+
dotFile.puts " }"
|
|
449
|
+
end
|
|
450
|
+
|
|
451
|
+
|
|
452
|
+
def labelTypes dotFile, edges
|
|
453
|
+
fromTos = Set.new
|
|
454
|
+
edges.each do |edge|
|
|
455
|
+
# fromTos.add "\"Alien Data Source\" -> \"Alien Data Source\""
|
|
456
|
+
fromTos.add "\"#{edge.from.type}\""
|
|
457
|
+
fromTos.add "\"#{edge.to.type}\""
|
|
458
|
+
end
|
|
459
|
+
return if fromTos.empty?
|
|
460
|
+
dotFile.puts "\n // 3--------------------------------------------------------------------"
|
|
461
|
+
dotFile.puts ' subgraph cluster_0 {'
|
|
462
|
+
dotFile.puts ' color=white;'
|
|
463
|
+
dotFile.puts ' node [shape="box3d" style="filled" ];'
|
|
464
|
+
fromTos.each do |ft|
|
|
465
|
+
dotFile.puts " #{ft}"
|
|
466
|
+
end
|
|
467
|
+
dotFile.puts ' }'
|
|
468
|
+
end
|
|
469
|
+
|
|
470
|
+
|
|
471
|
+
def emit(local=@localEmit, stuff)
|
|
472
|
+
# puts "\nstuff.class #{stuff.class} :: #{stuff}" if local
|
|
473
|
+
if stuff.is_a? String then
|
|
474
|
+
lines = stuff.split(/\n/)
|
|
475
|
+
lines.each do |line|
|
|
476
|
+
@logger.debug "#{@emitPrefix}#{line}"
|
|
477
|
+
puts "#{@emitPrefix}#{line}" if local
|
|
478
|
+
end
|
|
479
|
+
else
|
|
480
|
+
@logger.debug "#{@emitPrefix}#{stuff}"
|
|
481
|
+
puts "#{@emitPrefix}#{stuff}" if local
|
|
482
|
+
end
|
|
483
|
+
end
|
|
484
|
+
|
|
485
|
+
|
|
486
|
+
def initDot twb
|
|
487
|
+
dotFile = File.open("#{twb}#{@@processName}.dot",'w')
|
|
488
|
+
dotFile.puts @@dotHeader
|
|
489
|
+
return dotFile
|
|
490
|
+
end
|
|
491
|
+
|
|
492
|
+
def closeDot dotFile, twb
|
|
493
|
+
dotFile.puts ' '
|
|
494
|
+
dotFile.puts '// -------------------------------------------------------------'
|
|
495
|
+
dotFile.puts ' '
|
|
496
|
+
dotFile.puts ' subgraph cluster_1 {'
|
|
497
|
+
# dotFile.puts ' color=white;'
|
|
498
|
+
dotFile.puts ' style=invis;'
|
|
499
|
+
# dotFile.puts ' border=0;'
|
|
500
|
+
dotFile.puts ' node [border=blue];'
|
|
501
|
+
dotFile.puts ' '
|
|
502
|
+
dotFile.puts ' "" [style=invis]'
|
|
503
|
+
dotFile.puts " \"Tableau Tools\\nCalculated Fields Map\\nWorkbook '#{twb}'\\n#{Time.new.ctime}\" [penwidth=0]"
|
|
504
|
+
# dotFile.puts " \"Tableau Tools Workbook Calculated Fields Map\\n#{Time.new.ctime}\" -> \"\" [style=invis]"
|
|
505
|
+
dotFile.puts ' '
|
|
506
|
+
dotFile.puts ' }'
|
|
507
|
+
dotFile.puts ' '
|
|
508
|
+
dotFile.puts '}'
|
|
509
|
+
dotFile.close
|
|
510
|
+
end
|
|
511
|
+
|
|
512
|
+
|
|
513
|
+
def renderDot twb, dot, format
|
|
514
|
+
emit "Rendering DOT file\n - #{twb}\n - #{dot}\n - #{format}"
|
|
515
|
+
imageType = '-T' + format
|
|
516
|
+
imageFile = twb + @@processName + 'Graph.' + format
|
|
517
|
+
imageParam = '-o' + imageFile
|
|
518
|
+
emit "system #{@@gvDotLocation} #{imageType} #{imageParam} #{dot}"
|
|
519
|
+
# system @@gvDotLocation, imageType, imageParam, dot
|
|
520
|
+
@imageFiles << imageFile
|
|
521
|
+
return imageFile
|
|
522
|
+
end
|
|
523
|
+
|
|
524
|
+
end # class
|
|
525
|
+
|
|
526
|
+
end # module Analysis
|
|
527
|
+
end # module Twb
|