twb 2.2.1 → 3.7.2

Sign up to get free protection for your applications and to get access to all the features.
Files changed (46) hide show
  1. checksums.yaml +4 -4
  2. data/lib/twb.rb +13 -1
  3. data/lib/twb/action.rb +5 -1
  4. data/lib/twb/analysis/AnnotatedFieldsCSVEmitter.rb +3 -0
  5. data/lib/twb/analysis/CalculatedFields/CalculatedFieldsAnalyzer.rb +276 -287
  6. data/lib/twb/analysis/CalculatedFields/MarkdownEmitter.rb +48 -34
  7. data/lib/twb/analysis/DataSources/DataSourceFieldsCSVEmitter.rb +103 -103
  8. data/lib/twb/analysis/DataSources/googlesheetdatasourcesanalyzer.rb +79 -0
  9. data/lib/twb/analysis/DocumentedFieldsMarkdownEmitter.rb +1 -1
  10. data/lib/twb/analysis/Sheets/sheetfieldsanalyzer.rb +82 -0
  11. data/lib/twb/analysis/Sheets/sheetfiltersanalyzer.rb +214 -0
  12. data/lib/twb/calculatedfield.rb +20 -5
  13. data/lib/twb/codedfield.rb +87 -0
  14. data/lib/twb/columnfield.rb +21 -2
  15. data/lib/twb/connection.rb +33 -0
  16. data/lib/twb/dashboard.rb +5 -1
  17. data/lib/twb/datasource.rb +131 -20
  18. data/lib/twb/dbfield.rb +4 -0
  19. data/lib/twb/field.rb +5 -1
  20. data/lib/twb/fieldcalculation.rb +134 -78
  21. data/lib/twb/localfield.rb +5 -1
  22. data/lib/twb/mappedfield.rb +5 -1
  23. data/lib/twb/metadatafield.rb +5 -1
  24. data/lib/twb/storyboard.rb +5 -1
  25. data/lib/twb/tabclass.rb +71 -0
  26. data/lib/twb/tabtest.rb +31 -0
  27. data/lib/twb/tabtool.rb +63 -0
  28. data/lib/twb/twbcodedfield.rb +87 -0
  29. data/lib/twb/util/cypher.rb +112 -0
  30. data/lib/twb/util/cypherpython.rb +128 -0
  31. data/lib/twb/util/docprep.rb +46 -0
  32. data/lib/twb/util/fielddomainloader.rb +108 -0
  33. data/lib/twb/util/gml.rb +144 -0
  34. data/lib/twb/util/gmledge.rb +73 -0
  35. data/lib/twb/util/graph.rb +30 -0
  36. data/lib/twb/util/graphedge.rb +8 -9
  37. data/lib/twb/util/graphnode.rb +46 -29
  38. data/lib/twb/util/tabgraph.rb +30 -0
  39. data/lib/twb/window.rb +5 -1
  40. data/lib/twb/workbook.rb +18 -5
  41. data/lib/twb/worksheet.rb +5 -1
  42. data/test/fieldAliases.rb +10 -0
  43. data/test/testFieldAliases.rb +65 -0
  44. data/test/testFieldDomainLoaded.rb +14 -0
  45. data/test/testFieldDomainLoader.rb +131 -0
  46. metadata +22 -1
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: 0ac4493d38cdcff16c2d5cb96873ba8356e80271
4
- data.tar.gz: 8f77ffab827d962ec69aae1061d72978a65111ed
3
+ metadata.gz: fd921c477a3ab3ef1de872935a8057bd4fd5ea27
4
+ data.tar.gz: de2bc297a944a544bb0f251ac5cc9f972e1f763c
5
5
  SHA512:
6
- metadata.gz: 0f146956037bfdbd37161ee9a631dc86827afa4af34470199beb2553b152cdeb95201994b088a2028ac89cb5d54e728ff70af4c794e1f9d7a1a903d3af9ab5a2
7
- data.tar.gz: ef4a7e6e3435b0e3bfa4d9ba096f8f055ec40ffc865196f0923b04cbd9818a9047c62799fb0f505ae1d940269719c71446e0227c44d79f15285fef058770bed8
6
+ metadata.gz: 46becdab0ca9837b76ddd7ffc46e26433ab130bda120c5150ddce3575da04e80309874dfef9ca69531556b6185b46131889b62fe3f0a05e58b341be83ef04d56
7
+ data.tar.gz: 41f15a08b3b9971c4ca237ea5a166c212488c6e13a5ee3ca59613b8894d7b23d619e9d54fbc3c41280b4fc543c9109af1985befb92a7b42dc2bea00dc14f865a
data/lib/twb.rb CHANGED
@@ -13,6 +13,8 @@
13
13
  # You should have received a copy of the GNU General Public License
14
14
  # along with this program. If not, see <http://www.gnu.org/licenses/>.
15
15
 
16
+ require_relative 'twb/tabclass'
17
+ require_relative 'twb/tabtool'
16
18
  require_relative 'twb/dashboard'
17
19
  require_relative 'twb/datasource'
18
20
  require_relative 'twb/docdashboard'
@@ -26,6 +28,7 @@ require_relative 'twb/worksheet'
26
28
  require_relative 'twb/action'
27
29
  require_relative 'twb/columnfield.rb'
28
30
  require_relative 'twb/calculatedfield'
31
+ require_relative 'twb/codedfield'
29
32
  require_relative 'twb/fieldcalculation'
30
33
  require_relative 'twb/docdashboardimagevert'
31
34
  require_relative 'twb/docdashboardwebvert'
@@ -33,9 +36,15 @@ require_relative 'twb/util/twbDashSheetDataDotBuilder'
33
36
  require_relative 'twb/util/dotFileRenderer'
34
37
  require_relative 'twb/util/htmllistcollapsible'
35
38
  require_relative 'twb/util/xraydashboards'
39
+ require_relative 'twb/util/graph'
36
40
  require_relative 'twb/util/graphnode'
37
41
  require_relative 'twb/util/graphedge'
38
42
  require_relative 'twb/util/graphedges'
43
+ require_relative 'twb/util/gml'
44
+ require_relative 'twb/util/cypher'
45
+ require_relative 'twb/util/cypherpython'
46
+ require_relative 'twb/util/fielddomainloader'
47
+ require_relative 'twb/util/docprep'
39
48
  require_relative 'twb/analysis/documentedfieldsmarkdownemitter'
40
49
  require_relative 'twb/analysis/annotatedfieldsCSVEmitter'
41
50
  require_relative 'twb/analysis/calculatedfields/calculatedfieldsanalyzer'
@@ -43,10 +52,13 @@ require_relative 'twb/analysis/calculatedfields/markdownemitter'
43
52
  require_relative 'twb/analysis/calculatedfields/csvemitter'
44
53
  require_relative 'twb/analysis/datasources/DataSourceFieldsCSVEmitter'
45
54
  require_relative 'twb/analysis/datasources/DataSourceTableFieldsCSVEmitter'
55
+ require_relative 'twb/analysis/datasources/googlesheetdatasourcesanalyzer'
46
56
  require_relative 'twb/analysis/Sheets/WorksheetDataStructureCSVEmitter'
57
+ require_relative 'twb/analysis/Sheets/sheetfiltersanalyzer'
58
+ require_relative 'twb/analysis/Sheets/sheetfieldsanalyzer'
47
59
 
48
60
  # Represents Tableau Workbooks and their contents.
49
61
  #
50
62
  module Twb
51
- VERSION = '2.2.1'
63
+ VERSION = '3.7.2'
52
64
  end
@@ -18,7 +18,7 @@ require 'digest/md5'
18
18
 
19
19
  module Twb
20
20
 
21
- class Action
21
+ class Action < TabClass
22
22
 
23
23
  # Notes
24
24
  # Sources:
@@ -68,6 +68,10 @@ module Twb
68
68
  return self
69
69
  end
70
70
 
71
+ def id
72
+ @id ||= @id = @name.hash
73
+ end
74
+
71
75
  def process
72
76
 
73
77
  end
@@ -37,6 +37,7 @@ module Analysis
37
37
  @@csvFileName = @@csvFileType + '.csv'
38
38
 
39
39
  def initialize
40
+ # puts "AnnotatedFieldsCSVEmitter - initializing"
40
41
  @csvFile = CSV.open(@@csvFileName,'w')
41
42
  @csvFile << @@csvHeader
42
43
  @csvRecords = Set.new
@@ -63,6 +64,7 @@ module Analysis
63
64
  @twb = twb if twb.instance_of? Twb::Workbook
64
65
  @twb = Twb::Workbook.new(twb) if twb.instance_of? String
65
66
  raise ArgumentError.new("ERROR in Workbok processing: '#{twb}' must be a Workbook (class) or the name of a Workbook (String), is a #{twb.class} \n ") unless @twb.is_a? Twb::Workbook
67
+ # puts "Processing #{@twb.name}"
66
68
  # --
67
69
  dss = @twb.datasources
68
70
  dss.each do |ds|
@@ -78,6 +80,7 @@ module Analysis
78
80
  end # def processTwb twb
79
81
 
80
82
  def recordField field
83
+ # puts "- #{field.uiname} :: #{field} ::: #{field.node}"
81
84
  @fieldNum += 1
82
85
  @lineNum = 0
83
86
  field.comment.each do |line|
@@ -1,4 +1,4 @@
1
- # calculatedfieldsanalyzer.rb - this Ruby script Copyright 2017 Christopher Gerrard
1
+ # calculatedfieldsanalyzer.rb - this Ruby script Copyright 2017, 2018 Christopher Gerrard
2
2
  #
3
3
  # This program is free software: you can redistribute it and/or modify
4
4
  # it under the terms of the GNU General Public License as published by
@@ -17,14 +17,16 @@ require 'nokogiri'
17
17
  require 'twb'
18
18
  require 'set'
19
19
  require 'csv'
20
- require 'logger'
21
20
 
22
21
  module Twb
23
22
  module Analysis
24
23
 
25
- class CalculatedFieldsAnalyzer
24
+ class CalculatedFieldsAnalyzer < Twb::Util::Graph
26
25
 
27
- attr_reader :calculatedFieldsCount, :formulaFieldsCount, :dataFiles
26
+ include TabTool
27
+
28
+ attr_reader :calculatedFieldsCount, :formulaFieldsCount, :dataFiles
29
+ attr_accessor :ttdocdir
28
30
 
29
31
  @@ttlogfile = 'CalculatedFieldsAnalyzer.ttlog'
30
32
  @@gvDotLocation = 'C:\\tech\\graphviz\\Graphviz2.38\\bin\\dot.exe'
@@ -40,12 +42,20 @@ module Analysis
40
42
  'Class',
41
43
  'Scope Isolation',
42
44
  'Formula Length',
43
- 'Formula Code',
44
45
  'Formula',
46
+ 'Formula (tech)',
45
47
  'Formula Comments',
46
48
  'Formula LOD?'
47
49
  ]
48
50
 
51
+ @@calcLinesCSVFileName = 'TwbCalculatedFieldFormulaLines.csv'
52
+ @@calcLinesCSVFileHeader = ['Calc Field #',
53
+ 'Workbook', 'Workbook Dir',
54
+ 'Data Source', 'Data Source Caption', 'Data Source Name (tech)',
55
+ 'Field Name', 'Field Caption', 'Field Name (tech)',
56
+ 'Formula', 'Formula Line #', 'Formula Line'
57
+ ]
58
+
49
59
  @@formFieldsCSVFileName = 'TwbFormulaFields.csv'
50
60
  @@formFieldsCSVFileHeader = ['Rec #',
51
61
  'Workbook', 'Workbook Dir',
@@ -71,258 +81,234 @@ module Analysis
71
81
  DOTHEADER
72
82
 
73
83
  def initialize
84
+ #-- Logging setup --
85
+ @ttdocdir = 'ttdoc'
86
+ # logfile = docFile(@@ttlogfile)
87
+ # @logger = Logger.new(logfile)
88
+ # @logger.level = Logger::DEBUG
89
+ emit ' Initializing CalculatedFieldsAnalyzer'
90
+ #-- CSV records collectors
74
91
  @csvCalculatedFields = []
75
92
  @csvFormulaFields = []
76
- # @testFile = File.open('testCSV.csv','w')
77
- # @testFile.puts @@calcFieldsCSVFileHeader.inspect
78
- #-- Logging setup --
79
- @logger = Logger.new(@@ttlogfile)
80
- @logger.level = Logger::DEBUG
93
+ @csvFormulaLines = []
81
94
  #-- Counters setup --
82
- @twbCount = 0
83
- @calculatedFieldsCount = 0
84
- @formulaFieldsCount = 0
85
- # --
86
- @referencedFields = SortedSet.new
87
- # --
88
- @dataFiles = {'TwbCalculatedFields.csv' => 'Calculated Fields & their Formulas', 'TwbFormulaFields.csv' => 'Fields referenced in Formulas'}
89
- # --
90
- @localEmit = false
91
- emit "\n\nLogging activity to: #{File.basename(@@ttlogfile)}"
92
- @imageFiles = []
95
+ @twbCount = 0
96
+ @calculatedFieldsCount = 0
97
+ @formulaFieldsCount = 0
98
+ #--
99
+ @referencedFields = SortedSet.new
100
+ #--
101
+ emit "init CSV #{@@calcFieldsCSVFileName}"
102
+ @csvCF = File.open(docFile(@@calcFieldsCSVFileName), 'w')
103
+ @csvCF.puts @@calcFieldsCSVFileHeader.to_csv
104
+ # @csvCF.close
105
+ #--
106
+ emit "init CSV #{@@calcLinesCSVFileName}"
107
+ @csvCFLs = File.open(docFile(@@calcLinesCSVFileName), 'w')
108
+ @csvCFLs.puts @@calcLinesCSVFileHeader.to_csv
109
+ # @csvCFLs.close
110
+ #--
111
+ emit "init CSV #{@@formFieldsCSVFileName}"
112
+ @csvFF = File.open(docFile(@@formFieldsCSVFileName), 'w')
113
+ @csvFF.puts @@formFieldsCSVFileHeader.to_csv
114
+ # @csvFF.close
115
+ #--
116
+ @dataFiles = { @@calcFieldsCSVFileName => 'Calculated Fields & their Formulas',
117
+ @@calcLinesCSVFileName => 'Calculated Fields & individual Formula Lines',
118
+ @@formFieldsCSVFileName => 'Fields referenced in Formulas'
119
+ }
120
+ #--
121
+ @localEmit = false
122
+ @imageFiles = []
123
+ end
124
+
125
+ def initDocDir
126
+ return if $ttdocdir.nil?
127
+ return if ''.eql? $ttdocdir
128
+ return if Dir.exists?($ttdocdir)
129
+ if File.exists? $ttdocdir
130
+ $ttdocdir = nil
131
+ return
132
+ end
133
+ Dir.mkdir $ttdocdir
93
134
  end
94
135
 
95
- def processTWB twbWithDir
96
- twb = File.basename(twbWithDir)
97
- @twb = Twb::Workbook.new twbWithDir
98
- emit "- Workbook: #{twbWithDir}"
136
+ def docFile name
137
+ @ttdocdir.nil? ? name : "#{@ttdocdir}\\#{name}"
138
+ end
139
+
140
+ def processTWB workbook
141
+ @twb = workbook.is_a?(String) ? Twb::Workbook.new(workbook) : workbook
142
+ throw Exception unless @twb.is_a? Twb::Workbook
143
+ emit "- Workbook: #{workbook}"
99
144
  emit " version: #{@twb.version}"
100
- return if twbWithDir.end_with? == "Tableau Calculated Fields Analyses.twb"
101
- twbDir = File.dirname(File.expand_path(twbWithDir))
102
- edges = Set.new
103
- # -- processing
145
+ @twbDir = @twb.dir #File.dirname(File.expand_path(workbook))
146
+ @edges = Set.new
147
+ #-- processing
104
148
  dss = @twb.datasources
105
- puts " # data sources: #{dss.length}"
106
- twbRootFields = Set.new
149
+ # puts " # data sources: #{dss.length}"
150
+ @twbRootFields = Set.new
107
151
  @twbFields = {}
152
+ @nodes = Set.new
108
153
  dss.each do |ds|
109
- puts "\t\t - #{ds.uiname} \t\t #{ds.calculatedFields.length}"
110
- next if ds.Parameters? # don't process the Parameters data source
111
- ds.calculatedFields.each do |calcField|
112
- emit "HANDLING CALCULATED FIELD:: #{calcField}"
113
- emit " :: #{calcField.calculation.formula}"
114
- emit " :: #{calcField.calculation.formulaResolved}"
115
- dsTechName = ds.name
116
- dsCaption = ds.caption
117
- dsName = ds.uiname
118
- dsID = dsTechName + ':::' + dsName
119
- emit "\n\n "
120
- emit "======================================================"
121
- emit "======================================================"
122
- emit "======= DATA SOURCE: #{ds.uiname} ====== "
123
- emit "======================================================"
124
- emit "======================================================\n\n "
125
- dsGraphNode = Twb::Util::Graphnode.new(name: dsName, id: dsID, type: :TwbDataConnection, properties: {workbook: twbWithDir})
126
- emit "\t dsgnode: #{dsGraphNode}"
127
- fieldUINames = ds.fieldUINames
128
- emit "ds.calculatedFields :: nil? #{ds.calculatedFields.nil?}"
129
- end
154
+ # puts "\t\t - #{ds.uiname} \t\t #{ds.calculatedFields.length}"
155
+ next if ds.Parameters? # don't process the Parameters data source - Parameters' fields aren't Calculated fields for our purposes
156
+ # dataSourceNode = Twb::Util::Graphnode.new(name: ds.uiname, id: ds.id, type: ds, properties: {workbook: workbook})
157
+ # @nodes.add dataSourceNode
158
+ # ds.calculatedFields.each do |calcField|
159
+ # end
130
160
  processDataSource ds
131
161
  end
162
+ mapTwb
163
+ emitGml
164
+ @twbCount += 1
165
+ end
166
+
167
+ def emitGml
168
+ gml = Twb::Util::GML.new
169
+ gml.fileName = @twb.name
170
+ gml.nodes = @nodes
171
+ gml.edges = @edges
172
+ gml.render
132
173
  end
133
174
 
134
175
  def processDataSource ds
135
- emit "Datasource: '#{ds.uiname}' -> #{ds.Parameters?}"
136
- dsFields = {}
137
- @twbFields[ds.uiname] = dsFields
138
- # next if ds.Parameters? # don't process the Parameters data source
139
- # it requires special handling, has different XML structure
140
- #-- For tracking unreferenced (root) calculated fields = calculatedFields - referencedFields
141
- calculatedFields = SortedSet.new
142
- referencedFields = SortedSet.new
143
- #--
144
- dsTechName = ds.name
145
- dsCaption = ds.caption
146
- dsName = ds.uiname
147
- dsID = dsTechName + ':::' + dsName
148
- emit "\n\n "
149
- emit "======================================================"
150
- emit "======================================================"
151
176
  emit "======= DATA SOURCE: #{ds.uiname} ====== "
152
- emit "======================================================"
153
- emit "======================================================\n\n "
154
- dsGraphNode = Twb::Util::Graphnode.new(name: dsName, id: dsID, type: :TwbDataConnection, properties: {workbook: twbWithDir})
155
- emit "\t dsgnode: #{dsGraphNode}"
156
- fieldUINames = ds.fieldUINames
157
- emit "ds.calculatedFields :: nil? #{ds.calculatedFields.nil?}"
177
+ dsNodes = Set.new
178
+ dsEdges = Set.new
179
+ dsFields = {}
180
+ @twbFields[ds.uiname] = dsFields
181
+ calculatedFields = SortedSet.new
182
+ fieldFormulaLines = []
183
+ referencedFields = SortedSet.new
184
+ dataSourceNode = Twb::Util::Graphnode.new(name: ds.uiname, id: ds.id, type: ds, properties: {workbook: @twb.name})
185
+ @nodes.add dataSourceNode
186
+ #-- process Calculatred Fields
158
187
  ds.calculatedFields.each do |calcField|
159
- emit "HANDLING CALCULATED FIELD:: #{calcField}"
160
- emit '--'
161
- fldCaption, = calcField.caption
162
- fldTechName = calcField.name
163
- fldName = calcField.uiname
164
- dataType = calcField.datatype
165
- role = calcField.role
166
- type = calcField.type
167
- fieldID = fldTechName+'::'+dsName
168
- calculatedFields.add fieldID
169
- #--
170
- dsFields[fldName] = calcField
171
- srcGraphNode = Twb::Util::Graphnode.new(name: fldName, id: fieldID, type: :CalculatedField, properties: {:DataSource => dsName})
172
- dsFieldEdge = Twb::Util::Graphedge.new(from: dsGraphNode, to: srcGraphNode, relationship: 'contains')
173
- #--
174
- emit "\t srfnode: #{srcGraphNode} "
175
- emit "\t dsfedge: #{dsFieldEdge} "
176
- edges.add dsFieldEdge
188
+ calculatedFields.add calcField.id
189
+ dsFields[calcField.uiname] = calcField
190
+ calcFieldNode = Twb::Util::Graphnode.new(name: calcField.uiname, id: calcField.id, type: calcField, properties: {:DataSource => ds.uiname})
191
+ @nodes.add calcFieldNode
192
+ dsFieldEdge = Twb::Util::Graphedge.new(from: dataSourceNode, to: calcFieldNode, relationship: 'contains')
193
+ @edges.add dsFieldEdge
177
194
  calculation = calcField.calculation
178
- emit "calculation: f? %8s -> %s " % [calculation.has_formula, calculation.formula ]
179
195
  if calculation.has_formula
180
- formulaFlat = calculation.formulaFlat
181
- uiFormula = formulaFlat.gsub(' XX ',' ')
182
- formulaLOD = formulaFlat.upcase =~ /^[ ]*{[ ]*(INCLUDE|FIXED|EXCLUDE)/
183
- resolvedFields = calculation.resolvedFields
184
- resolvedFields.each do |rf|
185
- emit "\tRESOLVED FLD: #{rf.inspect}"
186
- calcFieldName = rf[:field]
187
- if rf[:source].nil?
188
- calcFieldRef = "[%s]" % [ calcFieldName ]
189
- dispFieldRef = "[%s]" % [ ds.fieldUIName(calcFieldName) ]
190
- else
191
- remoteDS = @twb.datasource(rf[:source])
192
- if remoteDS.nil?
193
- calcFieldRef = "[DS_NOT_FOUND].[%s]" % [ calcFieldName ]
194
- dispFieldRef = calcFieldRef
195
- else
196
- remoteDSName = remoteDS.uiname
197
- remoteDSFld = remoteDS.fieldUIName(calcFieldName)
198
- calcFieldRef = "[%s].[%s]" % [ rf[:source], calcFieldName ]
199
- dispFieldRef = "[%s].[%s]" % [ remoteDSName, remoteDSFld ]
200
- end # remoteDS.nil?
201
- end # rf[:source].nil?
202
- emit "\tcalcFieldRef: #{calcFieldRef}"
203
- emit "\tdispFieldRef: #{dispFieldRef}"
204
- uiFormula = uiFormula.gsub(calcFieldRef, dispFieldRef)
205
- end # resolvedFields.each
196
+ #-- collect field formulas as single lines
206
197
  @csvCalculatedFields.push [
207
198
  @calculatedFieldsCount += 1,
208
- twb,
209
- twbDir,
210
- dsName,
211
- dsCaption,
212
- dsTechName,
213
- fldName,
214
- fldCaption,
215
- fldTechName,
216
- dsTechName + '::' + fldTechName,
217
- dataType,
218
- role,
219
- type,
199
+ @twb.name,
200
+ @twbDir,
201
+ ds.uiname,
202
+ ds.caption,
203
+ ds.name,
204
+ calcField.uiname,
205
+ calcField.caption,
206
+ calcField.name,
207
+ ds.name + '::' + calcField.name,
208
+ calcField.datatype,
209
+ calcField.role,
210
+ calcField.type,
220
211
  calculation.class,
221
212
  calculation.scopeIsolation,
222
213
  calculation.formulaFlat.length,
223
- calculation.formulaFlat,
224
- uiFormula[0...200],
214
+ calculation.formulaResolved,
215
+ calculation.formulaFlat,
225
216
  calculation.comments,
226
- !formulaLOD.nil?
217
+ calculation.is_lod
227
218
  ]
228
- resolvedFields.each do |rf|
229
- emit "\t\t res field : #{rf[:field]} "
230
- emit "\t\t res source: #{rf[:source]}"
231
- calcFieldName = rf[:field]
232
- calcDataSource = rf[:source]
233
- localDataSource = rf[:source].nil? # if there isn't a rf[:source] value
234
- # the field is from this data source
235
- # else the field is from an alien data source (in the same workbook)
236
- refDataSource = localDataSource ? ds : @twb.datasource(calcDataSource) # data source may not be in Workbook
237
- refDataSourceName = !refDataSource.nil? ? refDataSource.uiname : calcDataSource + '\n***DATA CONNECTION NOT IN WORKBOOK***'
238
- if !refDataSource.nil?
239
- dispFieldName = refDataSource.fieldUIName(calcFieldName)
240
- calcFieldTable = refDataSource.fieldTable(calcFieldName)
241
- else
242
- dispFieldName = calcFieldName
243
- calcFieldTable = calcFieldName
244
- end
245
- emit "\t\t calc field : #{dispFieldName} nil?<#{dispFieldName.nil?}>"
246
- emit "\t\t data source: #{refDataSourceName}"
247
- emit "\t\t table: #{calcFieldTable} nil?<#{calcFieldTable.nil?}>"
248
- properties = {'DataSource' => dsName, 'DataSourceReference' => 'local'}
249
- if dispFieldName.nil?
250
- dispFieldName = "<#{calcFieldName}>::<#{calcDataSource}> UNDEFINED"
251
- properties['status'] = 'UNDEFINED'
252
- end
253
- calcFieldID = "#{calcFieldName}::#{refDataSourceName}"
254
- if !localDataSource
255
- calcFieldID = "#{calcFieldName}:LDS:#{ds.uiname}:RDS:#{refDataSourceName}"
256
- properties['DataSourceReference'] = 'remote'
257
- end
258
- calcFieldTable = ds.fieldTable(calcFieldName)
259
- calcFieldType = calcFieldTable.nil? ? :CalculatedField : :DatabaseField
260
- calcFieldNode = Twb::Util::Graphnode.new(name: dispFieldName, id: calcFieldID, type: calcFieldType, properties: properties)
261
- fieldFieldEdge = Twb::Util::Graphedge.new(from: srcGraphNode, to: calcFieldNode, relationship: 'references')
262
- edges.add fieldFieldEdge
263
- referencedFields.add calcFieldID
264
- emit "\t\t calcFieldNode: #{calcFieldNode}"
265
- emit "\t\t graphEdge: #{fieldFieldEdge}"
266
- fldToDsNode = calcFieldNode
267
- if !calcFieldTable.nil?
268
- tableID = calcFieldTable + ':::' + ds.uiname
269
- tableName = "-[#{calcFieldTable}]-"
270
- tableNode = Twb::Util::Graphnode.new(name: tableName, id: tableID, type: :DBTable, properties: properties)
271
- fieldFieldEdge = Twb::Util::Graphedge.new(from: calcFieldNode, to: tableNode, relationship: 'is a field in')
272
- edges.add fieldFieldEdge
273
- fldToDsNode = tableNode
219
+ #-- collect individual formula lines
220
+ flnum = 0
221
+ emit "@@ FL: #{calcField.uiname}"
222
+ calculation.formulaResolvedLines.each do |fl|
223
+ emit "@@ FL: => '#{fl}'"
224
+ fieldFormulaLines.push [ @calculatedFieldsCount, # 'Calc Field #',
225
+ @twb.name, # 'Workbook',
226
+ @twbDir, # 'Workbook Dir',
227
+ ds.uiname, # 'Data Source',
228
+ ds.caption, # 'Data Source Caption',
229
+ ds.name, # 'Data Source Name (tech)',
230
+ calcField.uiname, # 'Field Name',
231
+ calcField.caption, # 'Field Caption',
232
+ calcField.name, # 'Field Name (tech)',
233
+ calcField.calculation.formulaFlatResolved, # 'Formula'
234
+ flnum += 1, # 'Formula Line #',
235
+ fl.start_with?(" ") ? "'#{fl}" : fl # 'Formula Line' - THIS IS A STUPID HACK NEEDED BECAUSE TABLEAU STRIPS LEADING BLANKS FROM CSV VALUES
236
+ ]
237
+ end
238
+ #-- collect fields referenced in formula
239
+ calculation.calcFields.each do |rf|
240
+ emit " rf.name :'#{rf.name}'"
241
+ emit " rf.uiname:'#{rf.uiname}'"
242
+ properties = {'DataSource' => ds.uiname, 'DataSourceReference' => 'local', :source => rf}
243
+ refFieldNode = Twb::Util::Graphnode.new(name: rf.uiname, id: rf.id, type: rf.type, properties: properties)
244
+ @nodes.add refFieldNode
245
+ fieldFieldEdge = Twb::Util::Graphedge.new(from: calcFieldNode, to: refFieldNode, relationship: 'references')
246
+ @edges.add fieldFieldEdge
247
+ referencedFields.add rf.id
248
+ refFieldTable = ds.fieldTable(rf.name)
249
+ emit "refFieldTable.nil? : #{refFieldTable.nil?}"
250
+ unless refFieldTable.nil?
251
+ tableID = refFieldTable + ':::' + ds.uiname
252
+ tableName = "||#{refFieldTable}||"
253
+ tableNode = Twb::Util::Graphnode.new(name: tableName, id: tableID, type: :DBTable, properties: properties)
254
+ @nodes.add tableNode
255
+ fieldFieldEdge = Twb::Util::Graphedge.new(from: refFieldNode, to: tableNode, relationship: 'is a field in')
256
+ @edges.add fieldFieldEdge
257
+ # fldToDsNode = tableNode
274
258
  end
275
- if !localDataSource
276
- alienDSNode = Twb::Util::Graphnode.new( name: '==>' + refDataSourceName,
277
- id: "#{ds.uiname}::::=>#{refDataSourceName}",
278
- type: :DBTable,
279
- properties: {'Home Source' => dsName, 'Remote Source' => refDataSourceName}
280
- )
281
- fieldFieldEdge = Twb::Util::Graphedge.new(from: fldToDsNode, to: alienDSNode, relationship: 'In Remote Data Source')
282
- edges.add fieldFieldEdge
283
- end
284
-
285
- # @csvFormulaFields.push [ @formulaFieldsCount+=1,
286
-
287
-
259
+ @csvFormulaFields.push [
260
+ @formulaFieldsCount += 1,
261
+ @twb.name,
262
+ @twbDir,
263
+ ds.uiname,
264
+ calcField.uiname,
265
+ calculation.formulaFlat,
266
+ calculation.formulaResolved,
267
+ rf.name,
268
+ rf.uiname,
269
+ rf.id,
270
+ refFieldTable
271
+ ]
288
272
  end # resolvedFields.each do
289
273
  end # if calculation.has_formula
290
274
  end # ds.calculatedFields.each
291
275
 
292
276
  dsRootFields = calculatedFields - referencedFields
293
277
  @referencedFields.merge referencedFields
294
- #--
295
- emit "--\nCalculated Fields\n-----------------"
296
- calculatedFields.each { |f| emit f }
297
- emit "--\nReferenced Fields\n-----------------"
298
- referencedFields.each { |f| emit f }
299
- emit "--\nDS Root Fields\n-----------------"
300
- dsRootFields.each { |f| emit f }
301
- emit "--"
302
- # --
303
- twbRootFields.merge dsRootFields
304
- @twbCount += 1
305
- mapTwb twb, edges, twbRootFields
306
- graphEdges twb, edges
278
+ @twbRootFields.merge dsRootFields
279
+ cypher @twb.name
280
+ cypherPy @twb.name
307
281
  emit "#######################"
308
- #--
309
- csvCF = File.open(@@calcFieldsCSVFileName, 'w')
310
- csvCF.puts @@calcFieldsCSVFileHeader.to_csv
282
+ #-- record calculated fields
283
+ # @csvCF = File.open(docFile(@@calcFieldsCSVFileName), 'a')
284
+ # @csvCF.puts @@calcFieldsCSVFileHeader.to_csv
285
+ emit "@@ record calculated fields ds: #{ds.uiname}"
286
+ # @csvCF.open('a')
311
287
  @csvCalculatedFields.each do |r|
312
- csvCF.puts r.to_csv
288
+ @csvCF.puts r.to_csv
313
289
  end
314
- csvCF.close
315
- #--
316
- csvFF = File.open(@@formFieldsCSVFileName, 'w')
317
- csvFF.puts @@formFieldsCSVFileHeader.to_csv
290
+ #-- record individual formula lines
291
+ # @csvCFLs = File.open(docFile(@@calcLinesCSVFileName), 'a')
292
+ # @csvCFLs.puts @@calcLinesCSVFileHeader.to_csv
293
+ emit "@@ individual formula lines ds: #{ds.uiname}"
294
+ fieldFormulaLines.each do |ffl|
295
+ emit "@@@@ FFLA: #{ffl}"
296
+ emit "@@@@ FFLB: #{ffl.to_csv}"
297
+ emit "@@@@ FFLc: "
298
+ @csvCFLs.puts ffl.to_csv
299
+ end
300
+ #-- record formula-referenced fields
301
+ # @csvFF = File.open(docFile(@@formFieldsCSVFileName), 'a')
302
+ # @csvFF.puts @@formFieldsCSVFileHeader.to_csv
303
+ emit "@@ formula-referenced fields ds: #{ds.uiname}"
318
304
  @csvFormulaFields.each do |r|
319
- csvFF.puts r.to_csv
305
+ @csvFF.puts r.to_csv
320
306
  end
321
- csvFF.close
322
307
  #--
323
308
  return @imageFiles
324
309
  end # def processDataSource
325
310
 
311
+
326
312
  def emitCalcfield calcField
327
313
  emit "\t FIELD cap :: #{calcField.caption} "
328
314
  emit "\t tname:: #{calcField.name}"
@@ -330,9 +316,12 @@ DOTHEADER
330
316
  emit "\t formula:: #{calculation.formulaFlat}"
331
317
  end
332
318
 
333
- def mapTwb twb, edges, rootFields
334
- dotFile = initDot twb
335
- dotFileName = File.basename dotFile
319
+ def mapTwb
320
+ twb = @twb.name
321
+ rootFields = @twbRootFields
322
+ dotStuff = initDot twb
323
+ dotFile = dotStuff[:file]
324
+ dotFileName = dotStuff[:name]
336
325
  dotFile.puts "\n // subgraph cluster_1 {"
337
326
  dotFile.puts " // color= grey;"
338
327
  dotFile.puts ""
@@ -340,85 +329,94 @@ DOTHEADER
340
329
  # this two step process coalesces the edges into a unique set, avoiding duplicating the dot
341
330
  # file entries, and can be shrunk when graph edges expose the bits necessary for management by Set
342
331
  emit "\n========================\nLoading Edges\n========================\n From DC? Referenced? Edge \n %s %s %s" % ['--------', '-----------', '-'*45]
343
- edges.each do |e|
332
+ @edges.each do |e|
344
333
  # don't want to emit edge which is from a Data Connection to a
345
334
  # Calculated Field which is also referenced by another calculated field
346
335
  isFromDC = e.from.type == :TwbDataConnection
347
336
  isRefField = @referencedFields.include?(e.to.id)
348
337
  edgesAsStrings.add(e.dot) unless isFromDC && isRefField
338
+ # emit " ES #{e.dot}"
339
+ # emit " ES from #{e.from}"
340
+ # emit " ES to #{e.to}"
349
341
  end
350
342
  emit "------------------------\n "
351
343
  edgesAsStrings.each do |es|
352
344
  dotFile.puts " #{es}"
353
- emit " #{es}"
354
345
  end
355
346
  emit "========================\n "
356
347
  dotFile.puts ""
357
348
  dotFile.puts " // }"
358
- dotFile.puts "\n\n // 4--------------------------------------------------------------------"
359
- # "table::JIRA_HARVEST_Correspondence__c::Jira" [label="JIRA_HARVEST_Correspondence__c"]
360
- nodes = SortedSet.new
361
- edges.each do |e|
362
- nodes.add e.from.dotLabel
363
- nodes.add e.to.dotLabel
364
- end
365
- nodes.each do |n|
366
- dotFile.puts n
349
+ dotFile.puts "\n\n // 4 NODES --------------------------------------------------------------------"
350
+ @nodes.each do |n|
351
+ dotFile.puts n.dotLabel
367
352
  end
368
353
  dotFile.puts "\n\n // 5--------------------------------------------------------------------"
369
- emitTypes( edges, dotFile )
370
- rankRootFields( dotFile, rootFields )
354
+ emitTypes( dotFile )
371
355
  closeDot( dotFile, twb )
372
- # renderPng(twb.name,dotFileName)
373
- # renderPdf(twb.name,dotFileName)
356
+ emit "Rendering DOT file - #{twb}"
374
357
  renderDot(twb,dotFileName,'pdf')
375
358
  renderDot(twb,dotFileName,'png')
376
359
  renderDot(twb,dotFileName,'svg')
377
- emitEdges edges
360
+ # emitEdges
378
361
  end
379
362
 
363
+ def cypher twbName
364
+ cypher = Twb::Util::Cypher.new
365
+ cypher.fileName = "#{twbName}.calcFields"
366
+ cypher.nodes = @nodes
367
+ cypher.edges = @edges
368
+ cypher.render
369
+ end
380
370
 
381
- def graphEdges twb, edges
382
- graphFile = File.new(twb + '.cypher', 'w')
383
- # graphFile.puts "OKEY DOKE, graphing away"
384
- cypherCode = Set.new
385
- edges.each do |edge|
386
- cypherCode.add edge.from.cypherCreate
387
- cypherCode.add edge.to.cypherCreate
388
- cypherCode.add edge.cypherCreate
389
- end
390
- cypherCode.each do |cc|
391
- graphFile.puts cc
392
- end
393
- graphFile.puts "\nreturn *"
394
- graphFile.close unless graphFile.nil?
395
- @imageFiles << File.basename(graphFile)
371
+ def cypherPy twbName
372
+ cypher = Twb::Util::CypherPython.new
373
+ cypher.fileName = "#{twbName}.calcFields"
374
+ cypher.nodes = @nodes
375
+ cypher.edges = @edges
376
+ cypher.render
396
377
  end
397
378
 
398
- def emitEdges edges
379
+ # def graphEdges twb
380
+ # # graphFile = File.new(twb + '.cypher', 'w')
381
+ # # # graphFile.puts "OKEY DOKE, graphing away"
382
+ # # cypherCode = Set.new
383
+ # # @edges.each do |edge|
384
+ # # cypherCode.add edge.from.cypherCreate
385
+ # # cypherCode.add edge.to.cypherCreate
386
+ # # cypherCode.add edge.cypherCreate
387
+ # # end
388
+ # # cypherCode.each do |cc|
389
+ # # graphFile.puts cc
390
+ # # end
391
+ # # graphFile.puts "\nreturn *"
392
+ # # graphFile.close unless graphFile.nil?
393
+ # # @imageFiles << File.basename(graphFile)
394
+ # end
395
+
396
+ def emitEdges
399
397
  emit " %-15s %s" % ['type', 'Edge']
400
398
  emit " %-15s %s" % ['-'*15, '-'*35]
401
- edges.each do |edge|
399
+ @edges.each do |edge|
402
400
  emit " %-15s %s" % [edge.from.type, edge.from]
403
401
  emit " %-15s %s" % [edge.to.type, edge.to]
404
402
  emit "\n "
405
403
  end
406
404
  end
407
405
 
408
- def emitTypes edges, dotFile
406
+ def emitTypes dotFile
409
407
  typedNodes = {}
410
408
  dotFile.puts "\n\n // 2--------------------------------------------------------------------"
411
- edges.each do |edge|
409
+ @edges.each do |edge|
412
410
  emit " EDGE :: #{edge}"
413
411
  loadNodeType typedNodes, edge.from
414
412
  loadNodeType typedNodes, edge.to
415
413
  end
416
414
  typedNodes.each do |type, nodes|
417
- emit "+++++++++ typedNodes of '#{type}'' "
418
- nodes.each do |node|
419
- emit " -n- #{node}"
420
- end
421
- rankSame(dotFile, type, nodes) unless type == :CalculatedField
415
+ # emit "+++++++++ typedNodes of '#{type}'' "
416
+ # nodes.each do |node|
417
+ # emit " -n- #{node}"
418
+ # end
419
+ rankSame(dotFile, type, nodes) unless type.eql? 'CalculatedField' # == :CalculatedField
422
420
  end
423
421
  # labelTypes dotFile, edges
424
422
  end
@@ -429,12 +427,17 @@ DOTHEADER
429
427
  set[type].add node
430
428
  end
431
429
 
430
+ @@unrankedTypes = ['CalculationField']
432
431
  def rankSame dotFile, type, nodes
432
+ return if @@unrankedTypes.include? type.to_s
433
+ @lines = SortedSet.new
434
+ nodes.each do |node|
435
+ @lines << node.id
436
+ end
433
437
  dotFile.puts "\n // '#{type}' --------------------------------------------------------------------"
434
438
  dotFile.puts "\n {rank=same "
435
- # dotFile.puts " \"#{type}\" [shape=\"box3d\" style=\"filled\" ]" unless ''.eql? type # [shape=\"box3d\" style=\"filled\" ]\"" unless label.equal? ''
436
- nodes.each do |node|
437
- dotFile.puts " \"#{node.id}\""
439
+ @lines.each do |line|
440
+ dotFile.puts " \"#{line}\""
438
441
  end
439
442
  dotFile.puts " }"
440
443
  end
@@ -443,15 +446,16 @@ DOTHEADER
443
446
  dotFile.puts "\n // Unreferenced (root) Calculated Fields -----------------------------------------"
444
447
  dotFile.puts "\n {rank=same "
445
448
  dsRootFields.each do |rf|
449
+ emit "ROOT FIELD: #{rf.class} :: #{rf}"
446
450
  dotFile.puts " \"#{rf}\""
447
451
  end
448
452
  dotFile.puts " }"
449
453
  end
450
454
 
451
455
 
452
- def labelTypes dotFile, edges
456
+ def labelTypes dotFile
453
457
  fromTos = Set.new
454
- edges.each do |edge|
458
+ @edges.each do |edge|
455
459
  # fromTos.add "\"Alien Data Source\" -> \"Alien Data Source\""
456
460
  fromTos.add "\"#{edge.from.type}\""
457
461
  fromTos.add "\"#{edge.to.type}\""
@@ -467,26 +471,11 @@ DOTHEADER
467
471
  dotFile.puts ' }'
468
472
  end
469
473
 
470
-
471
- def emit(local=@localEmit, stuff)
472
- # puts "\nstuff.class #{stuff.class} :: #{stuff}" if local
473
- if stuff.is_a? String then
474
- lines = stuff.split(/\n/)
475
- lines.each do |line|
476
- @logger.debug "#{@emitPrefix}#{line}"
477
- puts "#{@emitPrefix}#{line}" if local
478
- end
479
- else
480
- @logger.debug "#{@emitPrefix}#{stuff}"
481
- puts "#{@emitPrefix}#{stuff}" if local
482
- end
483
- end
484
-
485
-
486
474
  def initDot twb
487
- dotFile = File.open("#{twb}#{@@processName}.dot",'w')
475
+ dotFileName = docFile("#{twb}#{@@processName}.dot")
476
+ dotFile = File.open(dotFileName,'w')
488
477
  dotFile.puts @@dotHeader
489
- return dotFile
478
+ return {:file => dotFile, :name => dotFileName}
490
479
  end
491
480
 
492
481
  def closeDot dotFile, twb
@@ -511,12 +500,12 @@ DOTHEADER
511
500
 
512
501
 
513
502
  def renderDot twb, dot, format
514
- emit "Rendering DOT file\n - #{twb}\n - #{dot}\n - #{format}"
515
503
  imageType = '-T' + format
516
- imageFile = twb + @@processName + 'Graph.' + format
517
- imageParam = '-o' + imageFile
518
- emit "system #{@@gvDotLocation} #{imageType} #{imageParam} #{dot}"
519
- # system @@gvDotLocation, imageType, imageParam, dot
504
+ imageFile = './ttdoc/' + twb + @@processName + 'Graph.' + format
505
+ imageParam = '-o"' + imageFile + '"'
506
+ emit "system #{@@gvDotLocation} #{imageType} #{imageParam} \"#{dot}\""
507
+ system "#{@@gvDotLocation} #{imageType} #{imageParam} \"#{dot}\""
508
+ emit " - #{imageFile}"
520
509
  @imageFiles << imageFile
521
510
  return imageFile
522
511
  end