twb 2.2.1 → 3.7.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (46) hide show
  1. checksums.yaml +4 -4
  2. data/lib/twb.rb +13 -1
  3. data/lib/twb/action.rb +5 -1
  4. data/lib/twb/analysis/AnnotatedFieldsCSVEmitter.rb +3 -0
  5. data/lib/twb/analysis/CalculatedFields/CalculatedFieldsAnalyzer.rb +276 -287
  6. data/lib/twb/analysis/CalculatedFields/MarkdownEmitter.rb +48 -34
  7. data/lib/twb/analysis/DataSources/DataSourceFieldsCSVEmitter.rb +103 -103
  8. data/lib/twb/analysis/DataSources/googlesheetdatasourcesanalyzer.rb +79 -0
  9. data/lib/twb/analysis/DocumentedFieldsMarkdownEmitter.rb +1 -1
  10. data/lib/twb/analysis/Sheets/sheetfieldsanalyzer.rb +82 -0
  11. data/lib/twb/analysis/Sheets/sheetfiltersanalyzer.rb +214 -0
  12. data/lib/twb/calculatedfield.rb +20 -5
  13. data/lib/twb/codedfield.rb +87 -0
  14. data/lib/twb/columnfield.rb +21 -2
  15. data/lib/twb/connection.rb +33 -0
  16. data/lib/twb/dashboard.rb +5 -1
  17. data/lib/twb/datasource.rb +131 -20
  18. data/lib/twb/dbfield.rb +4 -0
  19. data/lib/twb/field.rb +5 -1
  20. data/lib/twb/fieldcalculation.rb +134 -78
  21. data/lib/twb/localfield.rb +5 -1
  22. data/lib/twb/mappedfield.rb +5 -1
  23. data/lib/twb/metadatafield.rb +5 -1
  24. data/lib/twb/storyboard.rb +5 -1
  25. data/lib/twb/tabclass.rb +71 -0
  26. data/lib/twb/tabtest.rb +31 -0
  27. data/lib/twb/tabtool.rb +63 -0
  28. data/lib/twb/twbcodedfield.rb +87 -0
  29. data/lib/twb/util/cypher.rb +112 -0
  30. data/lib/twb/util/cypherpython.rb +128 -0
  31. data/lib/twb/util/docprep.rb +46 -0
  32. data/lib/twb/util/fielddomainloader.rb +108 -0
  33. data/lib/twb/util/gml.rb +144 -0
  34. data/lib/twb/util/gmledge.rb +73 -0
  35. data/lib/twb/util/graph.rb +30 -0
  36. data/lib/twb/util/graphedge.rb +8 -9
  37. data/lib/twb/util/graphnode.rb +46 -29
  38. data/lib/twb/util/tabgraph.rb +30 -0
  39. data/lib/twb/window.rb +5 -1
  40. data/lib/twb/workbook.rb +18 -5
  41. data/lib/twb/worksheet.rb +5 -1
  42. data/test/fieldAliases.rb +10 -0
  43. data/test/testFieldAliases.rb +65 -0
  44. data/test/testFieldDomainLoaded.rb +14 -0
  45. data/test/testFieldDomainLoader.rb +131 -0
  46. metadata +22 -1
@@ -1,4 +1,4 @@
1
- # calculatedfieldsanalyzer.rb - this Ruby script Copyright 2017 Christopher Gerrard
1
+ # calculatedfieldsanalyzer.rb - this Ruby script Copyright 2017, 2018 Christopher Gerrard
2
2
  #
3
3
  # This program is free software: you can redistribute it and/or modify
4
4
  # it under the terms of the GNU General Public License as published by
@@ -13,7 +13,7 @@
13
13
  # You should have received a copy of the GNU General Public License
14
14
  # along with this program. If not, see <http://www.gnu.org/licenses/>.
15
15
 
16
- require 'twb'
16
+ # require 'twb'
17
17
 
18
18
  module Twb
19
19
  module Analysis
@@ -24,46 +24,60 @@ module CalculatedFields
24
24
  attr_reader :docFileName
25
25
 
26
26
  def initialize
27
- # puts ""
27
+ #@docFile.puts ""
28
28
  end
29
29
 
30
30
  def processTwb twb
31
- twb = File.basename(twb)
32
- @twb = Twb::Workbook.new twb
33
- @docFileName = twb + '.CalculatedFields.md'
31
+ # twb = File.basename(twb)
32
+ @twb = twb #Twb::Workbook.new twb
33
+ @docFileName = './ttdoc/' + @twb.name + '.CalculatedFields.md'
34
34
  @docFile = File.open(@docFileName,'w')
35
- @docFile.puts "## #{twb}"
35
+ @docFile.puts "## #{twb.name}"
36
36
  dsNames = @twb.datasourceUINames
37
37
  @docFile.puts "#{dsNames.length} Data Sources"
38
- dsNames.each do |dsn|
39
- ds = @twb.datasource dsn
40
- # puts " => #{ds.uiname} "
41
- # end
42
- # dss = @twb.datasources
43
- # dss.each do |ds|
44
- @docFile.puts "### #{ds.uiname}"
45
- calcFields = ds.calculatedFieldsMap.sort_by { |fldName,calc| fldName }
46
- calcFields.each do |fldname, field|
47
- calculation = field.calculation
48
- @docFile.puts "\n##### #{fldname} "
49
- @docFile.puts "###### ...description..."
50
- @docFile.puts "```"
51
- if calculation.has_formula
52
- @docFile.puts calculation.formulaResolved
53
- @docFile.puts "\n -- Fields --" unless calculation.calcFields.empty?
54
- refFields = SortedSet.new
55
- calculation.calcFields.each do |cf|
56
- fds = if cf.dataSourceRef == :remote then "<<#{cf.dataSource}>>" else '' end
57
- refFields.add "#{cf.uiName} \t #{fds}"
58
- end
59
- refFields.each do |rf|
60
- @docFile.puts " #{rf}"
38
+
39
+
40
+ @twb.datasources.each do |ds|
41
+ @docFile.puts "# #{ds.uiname}"
42
+ @docFile.puts "__has #{ds.calculatedFields.length} calculated fields__\n "
43
+ cnt = 0
44
+ ds.calculatedFields.each do |cf|
45
+ cnt += 1
46
+ @docFile.puts "### '#{cf.uiname}'"
47
+ @docFile.puts "```"
48
+ cf.formulaResolvedLines.each do
49
+ |l|@docFile.puts "#{l.gsub('<<','[').gsub('>>',']')}"
61
50
  end
62
- @docFile.puts "```"
63
- end
51
+ @docFile.puts "```"
52
+ if cf.calcFields.length > 0
53
+ fieldsRefOrder = []
54
+ fieldsSortSet = SortedSet.new
55
+ cf.calcFields.each do |field|
56
+ fieldsRefOrder.push field.uiname
57
+ fieldsSortSet << field.uiname
58
+ end
59
+ if fieldsRefOrder != fieldsSortSet.to_a
60
+ @docFile.puts "```"
61
+ @docFile.puts "Fields - reference order:"
62
+ fieldsRefOrder.each do |field|
63
+ @docFile.puts " '#{field}'"
64
+ end
65
+ @docFile.puts "```"
66
+ end
67
+ if fieldsSortSet.length > 0
68
+ @docFile.puts "```"
69
+ @docFile.puts "Fields:"
70
+ fieldsSortSet.each do |field|
71
+ @docFile.puts " '#{field}'"
72
+ end
73
+ @docFile.puts "```"
74
+ end
75
+ end
76
+ @docFile.puts "\n "
64
77
  end
65
- end
66
- end
78
+ @docFile.puts "counted #{cnt} calculated fields\n "
79
+ end # twb.datasources.each
80
+ end # def processTwb twb
67
81
 
68
82
  end # class MarkdownEmitter
69
83
 
@@ -1,4 +1,4 @@
1
- # calculatedfieldsanalyzer.rb - this Ruby script Copyright 2017 Christopher Gerrard
1
+ # calculatedfieldsanalyzer.rb - this Ruby script Copyright 2017, 2018 Christopher Gerrard
2
2
  #
3
3
  # This program is free software: you can redistribute it and/or modify
4
4
  # it under the terms of the GNU General Public License as published by
@@ -25,35 +25,24 @@ module DataSources
25
25
  attr_reader :csvFileName, :csvRecords
26
26
  attr_reader :dsCount, :fieldsCount
27
27
 
28
- @@csvHeader = ['Record #',
29
- 'Workbook',
30
- 'Workbook Dir',
31
- 'Data Source',
32
- 'Field',
33
- 'Twb Type',
34
- 'Hidden', # props[:hidden],
35
- 'Type - Column', # props[:columnField],
36
- 'Type - Calculated', # props[:calculatedField],
37
- 'Type - Db', # props[:dbField],
38
- 'Type - Mapped', # props[:mappedField]
39
- 'Type - MetaData', # props[:mappedField]
40
- ]
41
- @@csvFileType = 'TwbDataSourceFields'
42
- @@csvFileName = @@csvFileType + '.csv'
43
-
44
- @@tallHeader = ['Field #',
45
- 'Workbook',
46
- 'Workbook Dir',
47
- 'Data Source',
48
- 'Field',
49
- 'Field Type',
50
- 'Property - Name',
51
- 'Property - Value'
52
- ]
28
+ # @@csvHeader = ['Record #',
29
+ # 'Workbook',
30
+ # 'Workbook Dir',
31
+ # 'Data Source',
32
+ # 'Field',
33
+ # 'Twb Type',
34
+ # 'Hidden', # props[:hidden],
35
+ # 'Type - Column', # props[:columnField],
36
+ # 'Type - Calculated', # props[:calculatedField],
37
+ # 'Type - Db', # props[:dbField],
38
+ # 'Type - Mapped', # props[:mappedField]
39
+ # 'Type - MetaData', # props[:mappedField]
40
+ # ]
41
+ # @@csvFileType = 'TwbDataSourceFields'
42
+ # @@csvFileName = @@csvFileType + '.csv'
43
+
53
44
  @@fullFileType = 'TwbDataSourceFieldsDetails'
54
45
  @@fullFileName = @@fullFileType + '.csv'
55
-
56
-
57
46
  @@fullHeader = ['Field #',
58
47
  'Workbook',
59
48
  'Workbook Dir',
@@ -65,22 +54,32 @@ module DataSources
65
54
  'Property - Name',
66
55
  'Property - Value'
67
56
  ]
68
- @@tallFileType = 'TwbDataSourceFieldsTech'
69
- @@tallFileName = @@tallFileType + '.csv'
57
+
58
+ # @@techFileType = 'TwbDataSourceFieldsTech'
59
+ # @@techFileName = @@techFileType + '.csv'
60
+ # @@techHeader = ['Field #',
61
+ # 'Workbook',
62
+ # 'Workbook Dir',
63
+ # 'Data Source',
64
+ # 'Field',
65
+ # 'Field Type',
66
+ # 'Property - Name',
67
+ # 'Property - Value'
68
+ # ]
70
69
 
71
70
  def initialize
72
- @csvFile = CSV.open(@@csvFileName,'w')
73
- @csvFile << @@csvHeader
74
- @csvRecords = Set.new
75
- puts "Opened: #{!@csvFile.nil?} - #{@@csvFileName}"
71
+ # @csvFile = CSV.open(@@csvFileName,'w')
72
+ # @csvFile << @@csvHeader
73
+ # @csvRecords = Set.new
74
+ # puts "Opened: #{!@csvFile.nil?} - #{@@csvFileName}"
76
75
  # --
77
76
  @csvFileFull = CSV.open(@@fullFileName,'w')
78
77
  @csvFileFull << @@fullHeader
79
78
  puts "Opened: #{!@csvFileFull.nil?} - #{@@fullFileName} "
80
79
  # --
81
- @csvFileTech = CSV.open(@@tallFileName,'w')
82
- @csvFileTech << @@tallHeader
83
- puts "Opened: #{!@csvFileTech.nil?} - #{@@tallFileName}"
80
+ # @csvFileTech = CSV.open(@@techFileName,'w')
81
+ # @csvFileTech << @@techHeader
82
+ # puts "Opened: #{!@csvFileTech.nil?} - #{@@techFileName}"
84
83
  # --
85
84
  @recNum = 0
86
85
  @dsCount = 0
@@ -106,11 +105,12 @@ module DataSources
106
105
  raise ArgumentError.new("ERROR in Workbok processing: '#{twb}' must be a Workbook (class) or the name of a Workbook (String), is a #{twb.class} \n ") unless @twb.is_a? Twb::Workbook
107
106
  # --
108
107
  dss = @twb.datasources
108
+ @fieldCnt = 0
109
109
  dss.each do |ds|
110
110
  @dsname = ds.uiname
111
111
  # puts "\n -- #{ds.uiname} "
112
112
  # tables = Set.new
113
- fields = {}
113
+ # fields = {}
114
114
  @dsCount += 1
115
115
  fclasses = Set.new
116
116
  ds.localFields.each do |field|
@@ -150,17 +150,17 @@ module DataSources
150
150
  # emitTech( ds, field.uiname, 'Mapped', field.properties)
151
151
  # recordTech( field, 'MappedA')
152
152
  end
153
- emitFields(fields)
153
+ # emitFields(fields)
154
154
  end
155
155
  end # def processTwb twb
156
156
 
157
157
  def recordFieldFull field, source
158
158
  # print field.properties.nil? ? '-' : ":#{field.properties.length}"
159
159
  # puts field.properties
160
- @recNum+=1
160
+ @fieldCnt+=1
161
161
  field.properties.each do |name,value|
162
162
  # print name
163
- @csvFileFull << [ @recNum,
163
+ @csvFileFull << [ @recNum+=1,
164
164
  @twb.name,
165
165
  @twb.dir,
166
166
  @dsname,
@@ -175,68 +175,68 @@ module DataSources
175
175
  end
176
176
  end
177
177
 
178
- def recordField fields, field, props
179
- # puts "%-65s :: %s " % [fieldName,props]
180
- return if field.uiname.nil?
181
- if fields.has_key? field.uiname
182
- fields[field.uiname].merge! field.properties
183
- else
184
- fields[field.uiname] = field.properties
185
- end
186
- end
187
-
188
- def emitFields fields
189
- fields.each do |fieldName,props|
190
- # puts "FIELD:: %-40s :: %s" % [fieldName,props.inspect]
191
- # class = props[]
192
- csvRec = [ @recNum+=1,
193
- @twb.name,
194
- @twb.dir,
195
- @dsname,
196
- fieldName,
197
- props[:type],
198
- props['hidden'],
199
- props[:columnField],
200
- props[:calculatedField],
201
- props[:dbField],
202
- props[:mappedField],
203
- props[:metadataField]
204
- ]
205
- @csvFile << csvRec
206
- @csvRecords.add csvRec
207
- end
208
- end
209
-
210
- def recordTech field, type
211
- @recNum+=1
212
- field.properties.each do |name,value|
213
- @csvFileTech << [ @recNum,
214
- @twb.name,
215
- @twb.dir,
216
- @dsname,
217
- field.uiname,
218
- type,
219
- name,
220
- value
221
- ]
222
- end
223
- end
224
-
225
- def emitTech dataSource, fieldName, type, properties
226
- # puts "XX #{dataSource.uiname} :: #{fieldName} #{} :: #{type} :: #{properties}"
227
- @recNum+=1
228
- properties.each do |name,value|
229
- @csvFileTech << [ @recNum,
230
- @twb.name,
231
- @twb.dir,
232
- dataSource.uiname,
233
- fieldName,
234
- type,
235
- name,
236
- value
237
- ]
238
- end
239
- end
178
+ # def recordField fields, field, props
179
+ # # puts "%-65s :: %s " % [fieldName,props]
180
+ # return if field.uiname.nil?
181
+ # if fields.has_key? field.uiname
182
+ # fields[field.uiname].merge! field.properties
183
+ # else
184
+ # fields[field.uiname] = field.properties
185
+ # end
186
+ # end
187
+
188
+ # def emitFields fields
189
+ # fields.each do |fieldName,props|
190
+ # # puts "FIELD:: %-40s :: %s" % [fieldName,props.inspect]
191
+ # # class = props[]
192
+ # csvRec = [ @recNum+=1,
193
+ # @twb.name,
194
+ # @twb.dir,
195
+ # @dsname,
196
+ # fieldName,
197
+ # props[:type],
198
+ # props['hidden'],
199
+ # props[:columnField],
200
+ # props[:calculatedField],
201
+ # props[:dbField],
202
+ # props[:mappedField],
203
+ # props[:metadataField]
204
+ # ]
205
+ # @csvFile << csvRec
206
+ # @csvRecords.add csvRec
207
+ # end
208
+ # end
209
+
210
+ # def recordTech field, type
211
+ # @recNum+=1
212
+ # field.properties.each do |name,value|
213
+ # @csvFileTech << [ @recNum,
214
+ # @twb.name,
215
+ # @twb.dir,
216
+ # @dsname,
217
+ # field.uiname,
218
+ # type,
219
+ # name,
220
+ # value
221
+ # ]
222
+ # end
223
+ # end
224
+
225
+ # def emitTech dataSource, fieldName, type, properties
226
+ # # puts "XX #{dataSource.uiname} :: #{fieldName} #{} :: #{type} :: #{properties}"
227
+ # @recNum+=1
228
+ # properties.each do |name,value|
229
+ # @csvFileTech << [ @recNum,
230
+ # @twb.name,
231
+ # @twb.dir,
232
+ # dataSource.uiname,
233
+ # fieldName,
234
+ # type,
235
+ # name,
236
+ # value
237
+ # ]
238
+ # end
239
+ # end
240
240
 
241
241
  def cleanup
242
242
  @csvFile.close unless @csvFile.nil?
@@ -0,0 +1,79 @@
1
+ # sheetfieldsanalyzer.rb Copyright (C) 2018 Chris Gerrard
2
+ #
3
+ # This program is free software: you can redistribute it and/or modify
4
+ # it under the terms of the GNU General Public License as published by
5
+ # the Free Software Foundation, either version 3 of the License, or
6
+ # (at your option) any later version.
7
+ #
8
+ # This program is distributed in the hope that it will be useful,
9
+ # but WITHOUT ANY WARRANTY; without even the implied warranty of
10
+ # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11
+ # GNU General Public License for more details.
12
+ #
13
+ # You should have received a copy of the GNU General Public License
14
+ # along with this program. If not, see <http://www.gnu.org/licenses/>.
15
+
16
+ require 'twb'
17
+ require 'csv'
18
+ require 'set'
19
+
20
+ module Twb
21
+ module Analysis
22
+
23
+ class GoogleSheetDataSourcesAnalyzer
24
+
25
+ include TabTool
26
+
27
+ attr_accessor :localEmit
28
+ attr_reader :twbname, :twbcount
29
+
30
+ def initialize
31
+ @twbcount = 0
32
+ @sheetCnt = 0
33
+ @filterCnt = 0
34
+ @funcdoc = {:class=>self.class, :blurb=>'Analyzing Google Sheet Data Sources from Tableau Workbooks.', :description=>nil,}
35
+ docFileName = docFile('TWBGoogleSheetDataSources.csv')
36
+ @csv = CSV.open(docFileName, 'w')
37
+ @csv << ["Workbook",'Data Source','Connection','File Name','Type','Table Name','Field']
38
+ @docfiles = [{:name=>docFileName,:desc=>"CSV File containing the data relating Google Sheet-based Data Sources."}]
39
+ end
40
+
41
+ def processTWB twb
42
+ if Twb::Workbook != twb.class
43
+ @twb = Twb::Workbook.new twb
44
+ else
45
+ @twb = twb
46
+ end
47
+ @twbname = @twb.name
48
+ emit "Workbook:: #{@twbname}"
49
+ @twbcount += 1
50
+ dss = twb.datasources
51
+ dss.each do |ds|
52
+ emit ds.uiname
53
+ conns = ds.node.xpath(".//connection[@class='google-sheets']")
54
+ if conns.length > 0
55
+ @relation = ds.node.at_xpath('./connection/relation')
56
+ @relName = @relation.attribute('name').text
57
+ @relType = @relation.attribute('type').text
58
+ @fileName = ds.node.at_xpath('.//named-connection/connection').attribute('filename')
59
+ emit "FILENAME: #{@fileName}"
60
+ tables = ds.node.xpath(".//relation[@type='table']")
61
+ # emit "# Tables: #{tables.length}"
62
+ tables.each do |table|
63
+ tableName = table.attribute('name')
64
+ columns = table.xpath('.//column')
65
+ columns.each do |column|
66
+ emit [@twbname,ds.uiname,@relName,@fileName,@relType,tableName,column.attribute('name')].to_csv
67
+ @csv << [@twbname,ds.uiname,@relName,@fileName,@relType,tableName,column.attribute('name')]
68
+ emit ''
69
+ end
70
+ end
71
+ end
72
+ emit " "
73
+ end
74
+ end
75
+
76
+ end #class SheetFieldsAnalyzer
77
+
78
+ end # module Analysis
79
+ end # module Twb
@@ -28,7 +28,7 @@ module Analysis
28
28
 
29
29
  def processTwb twbName
30
30
  @twb = Twb::Workbook.new twbName
31
- @docFileName = @twb.name + '.DocumentedFields.md'
31
+ @docFileName = '.ttdoc' + @twb.name + '.DocumentedFields.md'
32
32
  @docFile = File.open(@docFileName,'w')
33
33
  @docFile.puts "## #{@twb.name}"
34
34
  dss = @twb.datasources