twb 2.2.1 → 3.7.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/lib/twb.rb +13 -1
- data/lib/twb/action.rb +5 -1
- data/lib/twb/analysis/AnnotatedFieldsCSVEmitter.rb +3 -0
- data/lib/twb/analysis/CalculatedFields/CalculatedFieldsAnalyzer.rb +276 -287
- data/lib/twb/analysis/CalculatedFields/MarkdownEmitter.rb +48 -34
- data/lib/twb/analysis/DataSources/DataSourceFieldsCSVEmitter.rb +103 -103
- data/lib/twb/analysis/DataSources/googlesheetdatasourcesanalyzer.rb +79 -0
- data/lib/twb/analysis/DocumentedFieldsMarkdownEmitter.rb +1 -1
- data/lib/twb/analysis/Sheets/sheetfieldsanalyzer.rb +82 -0
- data/lib/twb/analysis/Sheets/sheetfiltersanalyzer.rb +214 -0
- data/lib/twb/calculatedfield.rb +20 -5
- data/lib/twb/codedfield.rb +87 -0
- data/lib/twb/columnfield.rb +21 -2
- data/lib/twb/connection.rb +33 -0
- data/lib/twb/dashboard.rb +5 -1
- data/lib/twb/datasource.rb +131 -20
- data/lib/twb/dbfield.rb +4 -0
- data/lib/twb/field.rb +5 -1
- data/lib/twb/fieldcalculation.rb +134 -78
- data/lib/twb/localfield.rb +5 -1
- data/lib/twb/mappedfield.rb +5 -1
- data/lib/twb/metadatafield.rb +5 -1
- data/lib/twb/storyboard.rb +5 -1
- data/lib/twb/tabclass.rb +71 -0
- data/lib/twb/tabtest.rb +31 -0
- data/lib/twb/tabtool.rb +63 -0
- data/lib/twb/twbcodedfield.rb +87 -0
- data/lib/twb/util/cypher.rb +112 -0
- data/lib/twb/util/cypherpython.rb +128 -0
- data/lib/twb/util/docprep.rb +46 -0
- data/lib/twb/util/fielddomainloader.rb +108 -0
- data/lib/twb/util/gml.rb +144 -0
- data/lib/twb/util/gmledge.rb +73 -0
- data/lib/twb/util/graph.rb +30 -0
- data/lib/twb/util/graphedge.rb +8 -9
- data/lib/twb/util/graphnode.rb +46 -29
- data/lib/twb/util/tabgraph.rb +30 -0
- data/lib/twb/window.rb +5 -1
- data/lib/twb/workbook.rb +18 -5
- data/lib/twb/worksheet.rb +5 -1
- data/test/fieldAliases.rb +10 -0
- data/test/testFieldAliases.rb +65 -0
- data/test/testFieldDomainLoaded.rb +14 -0
- data/test/testFieldDomainLoader.rb +131 -0
- metadata +22 -1
@@ -1,4 +1,4 @@
|
|
1
|
-
# calculatedfieldsanalyzer.rb - this Ruby script Copyright 2017 Christopher Gerrard
|
1
|
+
# calculatedfieldsanalyzer.rb - this Ruby script Copyright 2017, 2018 Christopher Gerrard
|
2
2
|
#
|
3
3
|
# This program is free software: you can redistribute it and/or modify
|
4
4
|
# it under the terms of the GNU General Public License as published by
|
@@ -13,7 +13,7 @@
|
|
13
13
|
# You should have received a copy of the GNU General Public License
|
14
14
|
# along with this program. If not, see <http://www.gnu.org/licenses/>.
|
15
15
|
|
16
|
-
require 'twb'
|
16
|
+
# require 'twb'
|
17
17
|
|
18
18
|
module Twb
|
19
19
|
module Analysis
|
@@ -24,46 +24,60 @@ module CalculatedFields
|
|
24
24
|
attr_reader :docFileName
|
25
25
|
|
26
26
|
def initialize
|
27
|
-
|
27
|
+
#@docFile.puts ""
|
28
28
|
end
|
29
29
|
|
30
30
|
def processTwb twb
|
31
|
-
twb = File.basename(twb)
|
32
|
-
@twb = Twb::Workbook.new twb
|
33
|
-
@docFileName = twb + '.CalculatedFields.md'
|
31
|
+
# twb = File.basename(twb)
|
32
|
+
@twb = twb #Twb::Workbook.new twb
|
33
|
+
@docFileName = './ttdoc/' + @twb.name + '.CalculatedFields.md'
|
34
34
|
@docFile = File.open(@docFileName,'w')
|
35
|
-
@docFile.puts "## #{twb}"
|
35
|
+
@docFile.puts "## #{twb.name}"
|
36
36
|
dsNames = @twb.datasourceUINames
|
37
37
|
@docFile.puts "#{dsNames.length} Data Sources"
|
38
|
-
|
39
|
-
|
40
|
-
|
41
|
-
|
42
|
-
|
43
|
-
|
44
|
-
|
45
|
-
|
46
|
-
|
47
|
-
|
48
|
-
|
49
|
-
|
50
|
-
@docFile.puts "```"
|
51
|
-
if calculation.has_formula
|
52
|
-
@docFile.puts calculation.formulaResolved
|
53
|
-
@docFile.puts "\n -- Fields --" unless calculation.calcFields.empty?
|
54
|
-
refFields = SortedSet.new
|
55
|
-
calculation.calcFields.each do |cf|
|
56
|
-
fds = if cf.dataSourceRef == :remote then "<<#{cf.dataSource}>>" else '' end
|
57
|
-
refFields.add "#{cf.uiName} \t #{fds}"
|
58
|
-
end
|
59
|
-
refFields.each do |rf|
|
60
|
-
@docFile.puts " #{rf}"
|
38
|
+
|
39
|
+
|
40
|
+
@twb.datasources.each do |ds|
|
41
|
+
@docFile.puts "# #{ds.uiname}"
|
42
|
+
@docFile.puts "__has #{ds.calculatedFields.length} calculated fields__\n "
|
43
|
+
cnt = 0
|
44
|
+
ds.calculatedFields.each do |cf|
|
45
|
+
cnt += 1
|
46
|
+
@docFile.puts "### '#{cf.uiname}'"
|
47
|
+
@docFile.puts "```"
|
48
|
+
cf.formulaResolvedLines.each do
|
49
|
+
|l|@docFile.puts "#{l.gsub('<<','[').gsub('>>',']')}"
|
61
50
|
end
|
62
|
-
|
63
|
-
|
51
|
+
@docFile.puts "```"
|
52
|
+
if cf.calcFields.length > 0
|
53
|
+
fieldsRefOrder = []
|
54
|
+
fieldsSortSet = SortedSet.new
|
55
|
+
cf.calcFields.each do |field|
|
56
|
+
fieldsRefOrder.push field.uiname
|
57
|
+
fieldsSortSet << field.uiname
|
58
|
+
end
|
59
|
+
if fieldsRefOrder != fieldsSortSet.to_a
|
60
|
+
@docFile.puts "```"
|
61
|
+
@docFile.puts "Fields - reference order:"
|
62
|
+
fieldsRefOrder.each do |field|
|
63
|
+
@docFile.puts " '#{field}'"
|
64
|
+
end
|
65
|
+
@docFile.puts "```"
|
66
|
+
end
|
67
|
+
if fieldsSortSet.length > 0
|
68
|
+
@docFile.puts "```"
|
69
|
+
@docFile.puts "Fields:"
|
70
|
+
fieldsSortSet.each do |field|
|
71
|
+
@docFile.puts " '#{field}'"
|
72
|
+
end
|
73
|
+
@docFile.puts "```"
|
74
|
+
end
|
75
|
+
end
|
76
|
+
@docFile.puts "\n "
|
64
77
|
end
|
65
|
-
|
66
|
-
|
78
|
+
@docFile.puts "counted #{cnt} calculated fields\n "
|
79
|
+
end # twb.datasources.each
|
80
|
+
end # def processTwb twb
|
67
81
|
|
68
82
|
end # class MarkdownEmitter
|
69
83
|
|
@@ -1,4 +1,4 @@
|
|
1
|
-
# calculatedfieldsanalyzer.rb - this Ruby script Copyright 2017 Christopher Gerrard
|
1
|
+
# calculatedfieldsanalyzer.rb - this Ruby script Copyright 2017, 2018 Christopher Gerrard
|
2
2
|
#
|
3
3
|
# This program is free software: you can redistribute it and/or modify
|
4
4
|
# it under the terms of the GNU General Public License as published by
|
@@ -25,35 +25,24 @@ module DataSources
|
|
25
25
|
attr_reader :csvFileName, :csvRecords
|
26
26
|
attr_reader :dsCount, :fieldsCount
|
27
27
|
|
28
|
-
@@csvHeader = ['Record #',
|
29
|
-
|
30
|
-
|
31
|
-
|
32
|
-
|
33
|
-
|
34
|
-
|
35
|
-
|
36
|
-
|
37
|
-
|
38
|
-
|
39
|
-
|
40
|
-
|
41
|
-
@@csvFileType = 'TwbDataSourceFields'
|
42
|
-
@@csvFileName = @@csvFileType + '.csv'
|
43
|
-
|
44
|
-
@@tallHeader = ['Field #',
|
45
|
-
'Workbook',
|
46
|
-
'Workbook Dir',
|
47
|
-
'Data Source',
|
48
|
-
'Field',
|
49
|
-
'Field Type',
|
50
|
-
'Property - Name',
|
51
|
-
'Property - Value'
|
52
|
-
]
|
28
|
+
# @@csvHeader = ['Record #',
|
29
|
+
# 'Workbook',
|
30
|
+
# 'Workbook Dir',
|
31
|
+
# 'Data Source',
|
32
|
+
# 'Field',
|
33
|
+
# 'Twb Type',
|
34
|
+
# 'Hidden', # props[:hidden],
|
35
|
+
# 'Type - Column', # props[:columnField],
|
36
|
+
# 'Type - Calculated', # props[:calculatedField],
|
37
|
+
# 'Type - Db', # props[:dbField],
|
38
|
+
# 'Type - Mapped', # props[:mappedField]
|
39
|
+
# 'Type - MetaData', # props[:mappedField]
|
40
|
+
# ]
|
41
|
+
# @@csvFileType = 'TwbDataSourceFields'
|
42
|
+
# @@csvFileName = @@csvFileType + '.csv'
|
43
|
+
|
53
44
|
@@fullFileType = 'TwbDataSourceFieldsDetails'
|
54
45
|
@@fullFileName = @@fullFileType + '.csv'
|
55
|
-
|
56
|
-
|
57
46
|
@@fullHeader = ['Field #',
|
58
47
|
'Workbook',
|
59
48
|
'Workbook Dir',
|
@@ -65,22 +54,32 @@ module DataSources
|
|
65
54
|
'Property - Name',
|
66
55
|
'Property - Value'
|
67
56
|
]
|
68
|
-
|
69
|
-
@@
|
57
|
+
|
58
|
+
# @@techFileType = 'TwbDataSourceFieldsTech'
|
59
|
+
# @@techFileName = @@techFileType + '.csv'
|
60
|
+
# @@techHeader = ['Field #',
|
61
|
+
# 'Workbook',
|
62
|
+
# 'Workbook Dir',
|
63
|
+
# 'Data Source',
|
64
|
+
# 'Field',
|
65
|
+
# 'Field Type',
|
66
|
+
# 'Property - Name',
|
67
|
+
# 'Property - Value'
|
68
|
+
# ]
|
70
69
|
|
71
70
|
def initialize
|
72
|
-
@csvFile = CSV.open(@@csvFileName,'w')
|
73
|
-
@csvFile << @@csvHeader
|
74
|
-
@csvRecords = Set.new
|
75
|
-
puts "Opened: #{!@csvFile.nil?} - #{@@csvFileName}"
|
71
|
+
# @csvFile = CSV.open(@@csvFileName,'w')
|
72
|
+
# @csvFile << @@csvHeader
|
73
|
+
# @csvRecords = Set.new
|
74
|
+
# puts "Opened: #{!@csvFile.nil?} - #{@@csvFileName}"
|
76
75
|
# --
|
77
76
|
@csvFileFull = CSV.open(@@fullFileName,'w')
|
78
77
|
@csvFileFull << @@fullHeader
|
79
78
|
puts "Opened: #{!@csvFileFull.nil?} - #{@@fullFileName} "
|
80
79
|
# --
|
81
|
-
@csvFileTech = CSV.open(@@
|
82
|
-
@csvFileTech << @@
|
83
|
-
puts "Opened: #{!@csvFileTech.nil?} - #{@@
|
80
|
+
# @csvFileTech = CSV.open(@@techFileName,'w')
|
81
|
+
# @csvFileTech << @@techHeader
|
82
|
+
# puts "Opened: #{!@csvFileTech.nil?} - #{@@techFileName}"
|
84
83
|
# --
|
85
84
|
@recNum = 0
|
86
85
|
@dsCount = 0
|
@@ -106,11 +105,12 @@ module DataSources
|
|
106
105
|
raise ArgumentError.new("ERROR in Workbok processing: '#{twb}' must be a Workbook (class) or the name of a Workbook (String), is a #{twb.class} \n ") unless @twb.is_a? Twb::Workbook
|
107
106
|
# --
|
108
107
|
dss = @twb.datasources
|
108
|
+
@fieldCnt = 0
|
109
109
|
dss.each do |ds|
|
110
110
|
@dsname = ds.uiname
|
111
111
|
# puts "\n -- #{ds.uiname} "
|
112
112
|
# tables = Set.new
|
113
|
-
fields = {}
|
113
|
+
# fields = {}
|
114
114
|
@dsCount += 1
|
115
115
|
fclasses = Set.new
|
116
116
|
ds.localFields.each do |field|
|
@@ -150,17 +150,17 @@ module DataSources
|
|
150
150
|
# emitTech( ds, field.uiname, 'Mapped', field.properties)
|
151
151
|
# recordTech( field, 'MappedA')
|
152
152
|
end
|
153
|
-
emitFields(fields)
|
153
|
+
# emitFields(fields)
|
154
154
|
end
|
155
155
|
end # def processTwb twb
|
156
156
|
|
157
157
|
def recordFieldFull field, source
|
158
158
|
# print field.properties.nil? ? '-' : ":#{field.properties.length}"
|
159
159
|
# puts field.properties
|
160
|
-
@
|
160
|
+
@fieldCnt+=1
|
161
161
|
field.properties.each do |name,value|
|
162
162
|
# print name
|
163
|
-
@csvFileFull << [ @recNum,
|
163
|
+
@csvFileFull << [ @recNum+=1,
|
164
164
|
@twb.name,
|
165
165
|
@twb.dir,
|
166
166
|
@dsname,
|
@@ -175,68 +175,68 @@ module DataSources
|
|
175
175
|
end
|
176
176
|
end
|
177
177
|
|
178
|
-
def recordField fields, field, props
|
179
|
-
|
180
|
-
|
181
|
-
|
182
|
-
|
183
|
-
|
184
|
-
|
185
|
-
|
186
|
-
end
|
187
|
-
|
188
|
-
def emitFields fields
|
189
|
-
|
190
|
-
|
191
|
-
|
192
|
-
|
193
|
-
|
194
|
-
|
195
|
-
|
196
|
-
|
197
|
-
|
198
|
-
|
199
|
-
|
200
|
-
|
201
|
-
|
202
|
-
|
203
|
-
|
204
|
-
|
205
|
-
|
206
|
-
|
207
|
-
|
208
|
-
end
|
209
|
-
|
210
|
-
def recordTech field, type
|
211
|
-
|
212
|
-
|
213
|
-
|
214
|
-
|
215
|
-
|
216
|
-
|
217
|
-
|
218
|
-
|
219
|
-
|
220
|
-
|
221
|
-
|
222
|
-
|
223
|
-
end
|
224
|
-
|
225
|
-
def emitTech dataSource, fieldName, type, properties
|
226
|
-
|
227
|
-
|
228
|
-
|
229
|
-
|
230
|
-
|
231
|
-
|
232
|
-
|
233
|
-
|
234
|
-
|
235
|
-
|
236
|
-
|
237
|
-
|
238
|
-
|
239
|
-
end
|
178
|
+
# def recordField fields, field, props
|
179
|
+
# # puts "%-65s :: %s " % [fieldName,props]
|
180
|
+
# return if field.uiname.nil?
|
181
|
+
# if fields.has_key? field.uiname
|
182
|
+
# fields[field.uiname].merge! field.properties
|
183
|
+
# else
|
184
|
+
# fields[field.uiname] = field.properties
|
185
|
+
# end
|
186
|
+
# end
|
187
|
+
|
188
|
+
# def emitFields fields
|
189
|
+
# fields.each do |fieldName,props|
|
190
|
+
# # puts "FIELD:: %-40s :: %s" % [fieldName,props.inspect]
|
191
|
+
# # class = props[]
|
192
|
+
# csvRec = [ @recNum+=1,
|
193
|
+
# @twb.name,
|
194
|
+
# @twb.dir,
|
195
|
+
# @dsname,
|
196
|
+
# fieldName,
|
197
|
+
# props[:type],
|
198
|
+
# props['hidden'],
|
199
|
+
# props[:columnField],
|
200
|
+
# props[:calculatedField],
|
201
|
+
# props[:dbField],
|
202
|
+
# props[:mappedField],
|
203
|
+
# props[:metadataField]
|
204
|
+
# ]
|
205
|
+
# @csvFile << csvRec
|
206
|
+
# @csvRecords.add csvRec
|
207
|
+
# end
|
208
|
+
# end
|
209
|
+
|
210
|
+
# def recordTech field, type
|
211
|
+
# @recNum+=1
|
212
|
+
# field.properties.each do |name,value|
|
213
|
+
# @csvFileTech << [ @recNum,
|
214
|
+
# @twb.name,
|
215
|
+
# @twb.dir,
|
216
|
+
# @dsname,
|
217
|
+
# field.uiname,
|
218
|
+
# type,
|
219
|
+
# name,
|
220
|
+
# value
|
221
|
+
# ]
|
222
|
+
# end
|
223
|
+
# end
|
224
|
+
|
225
|
+
# def emitTech dataSource, fieldName, type, properties
|
226
|
+
# # puts "XX #{dataSource.uiname} :: #{fieldName} #{} :: #{type} :: #{properties}"
|
227
|
+
# @recNum+=1
|
228
|
+
# properties.each do |name,value|
|
229
|
+
# @csvFileTech << [ @recNum,
|
230
|
+
# @twb.name,
|
231
|
+
# @twb.dir,
|
232
|
+
# dataSource.uiname,
|
233
|
+
# fieldName,
|
234
|
+
# type,
|
235
|
+
# name,
|
236
|
+
# value
|
237
|
+
# ]
|
238
|
+
# end
|
239
|
+
# end
|
240
240
|
|
241
241
|
def cleanup
|
242
242
|
@csvFile.close unless @csvFile.nil?
|
@@ -0,0 +1,79 @@
|
|
1
|
+
# sheetfieldsanalyzer.rb Copyright (C) 2018 Chris Gerrard
|
2
|
+
#
|
3
|
+
# This program is free software: you can redistribute it and/or modify
|
4
|
+
# it under the terms of the GNU General Public License as published by
|
5
|
+
# the Free Software Foundation, either version 3 of the License, or
|
6
|
+
# (at your option) any later version.
|
7
|
+
#
|
8
|
+
# This program is distributed in the hope that it will be useful,
|
9
|
+
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
10
|
+
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
11
|
+
# GNU General Public License for more details.
|
12
|
+
#
|
13
|
+
# You should have received a copy of the GNU General Public License
|
14
|
+
# along with this program. If not, see <http://www.gnu.org/licenses/>.
|
15
|
+
|
16
|
+
require 'twb'
|
17
|
+
require 'csv'
|
18
|
+
require 'set'
|
19
|
+
|
20
|
+
module Twb
|
21
|
+
module Analysis
|
22
|
+
|
23
|
+
class GoogleSheetDataSourcesAnalyzer
|
24
|
+
|
25
|
+
include TabTool
|
26
|
+
|
27
|
+
attr_accessor :localEmit
|
28
|
+
attr_reader :twbname, :twbcount
|
29
|
+
|
30
|
+
def initialize
|
31
|
+
@twbcount = 0
|
32
|
+
@sheetCnt = 0
|
33
|
+
@filterCnt = 0
|
34
|
+
@funcdoc = {:class=>self.class, :blurb=>'Analyzing Google Sheet Data Sources from Tableau Workbooks.', :description=>nil,}
|
35
|
+
docFileName = docFile('TWBGoogleSheetDataSources.csv')
|
36
|
+
@csv = CSV.open(docFileName, 'w')
|
37
|
+
@csv << ["Workbook",'Data Source','Connection','File Name','Type','Table Name','Field']
|
38
|
+
@docfiles = [{:name=>docFileName,:desc=>"CSV File containing the data relating Google Sheet-based Data Sources."}]
|
39
|
+
end
|
40
|
+
|
41
|
+
def processTWB twb
|
42
|
+
if Twb::Workbook != twb.class
|
43
|
+
@twb = Twb::Workbook.new twb
|
44
|
+
else
|
45
|
+
@twb = twb
|
46
|
+
end
|
47
|
+
@twbname = @twb.name
|
48
|
+
emit "Workbook:: #{@twbname}"
|
49
|
+
@twbcount += 1
|
50
|
+
dss = twb.datasources
|
51
|
+
dss.each do |ds|
|
52
|
+
emit ds.uiname
|
53
|
+
conns = ds.node.xpath(".//connection[@class='google-sheets']")
|
54
|
+
if conns.length > 0
|
55
|
+
@relation = ds.node.at_xpath('./connection/relation')
|
56
|
+
@relName = @relation.attribute('name').text
|
57
|
+
@relType = @relation.attribute('type').text
|
58
|
+
@fileName = ds.node.at_xpath('.//named-connection/connection').attribute('filename')
|
59
|
+
emit "FILENAME: #{@fileName}"
|
60
|
+
tables = ds.node.xpath(".//relation[@type='table']")
|
61
|
+
# emit "# Tables: #{tables.length}"
|
62
|
+
tables.each do |table|
|
63
|
+
tableName = table.attribute('name')
|
64
|
+
columns = table.xpath('.//column')
|
65
|
+
columns.each do |column|
|
66
|
+
emit [@twbname,ds.uiname,@relName,@fileName,@relType,tableName,column.attribute('name')].to_csv
|
67
|
+
@csv << [@twbname,ds.uiname,@relName,@fileName,@relType,tableName,column.attribute('name')]
|
68
|
+
emit ''
|
69
|
+
end
|
70
|
+
end
|
71
|
+
end
|
72
|
+
emit " "
|
73
|
+
end
|
74
|
+
end
|
75
|
+
|
76
|
+
end #class SheetFieldsAnalyzer
|
77
|
+
|
78
|
+
end # module Analysis
|
79
|
+
end # module Twb
|
@@ -28,7 +28,7 @@ module Analysis
|
|
28
28
|
|
29
29
|
def processTwb twbName
|
30
30
|
@twb = Twb::Workbook.new twbName
|
31
|
-
@docFileName = @twb.name + '.DocumentedFields.md'
|
31
|
+
@docFileName = '.ttdoc' + @twb.name + '.DocumentedFields.md'
|
32
32
|
@docFile = File.open(@docFileName,'w')
|
33
33
|
@docFile.puts "## #{@twb.name}"
|
34
34
|
dss = @twb.datasources
|