twb 3.9.7 → 4.3.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 8a3bca661db5662df120e34443a68874a5a0d5f6
-  data.tar.gz: 56e2f1171c7c4df1b64a6f4767070ecb9f023b76
+  metadata.gz: 896e3b1f94079e230c28b1f65553ac5b870c3263
+  data.tar.gz: 318d27a633debcdf2c3eb1c323017e6d227e569e
 SHA512:
-  metadata.gz: 85fc520db33e97945985b457870f21f73ced2105ba7b60287ccefb09291292da2969f364a771d9d6283abd8b00b61a58f43864500dc8fd5c7e19fcfc1a73a1f4
-  data.tar.gz: 55834df4440a51aad3c9ed05aa069c961365aafa56254aaf99a59cd64734cdbf0c8ca1aa3cd9e5d64fbce7883380ebf116d4ee9ffbd58ac414dddb6fac2f6be2
+  metadata.gz: af1fe8b04a2d323252026d718f64853489e80d0db24e6255e377480d51102e928215b40eda700d474d06b795aece2858759a0aaab4d87cf93b9e47c45fabd585
+  data.tar.gz: 0d79ef7cc04e9adfd6c075cdf776f6ffa1a1d4b4dfd7d755f3eff2dc879a4783ea580e5499d10b5bee724ebd274529fdfa46c40929d125b75cb998b06c720431

data/lib/twb.rb

@@ -17,6 +17,7 @@ require_relative 'twb/tabclass'
 require_relative 'twb/tabtool'
 require_relative 'twb/dashboard'
 require_relative 'twb/datasource'
+require_relative 'twb/connection'
 require_relative 'twb/docdashboard'
 require_relative 'twb/dbfield'
 require_relative 'twb/localfield'
@@ -56,16 +57,18 @@ require_relative 'twb/analysis/calculatedfields/markdownemitter'
 require_relative 'twb/analysis/calculatedfields/csvemitter'
 require_relative 'twb/analysis/datasources/DataSourceFieldsCSVEmitter'
 require_relative 'twb/analysis/datasources/DataSourceTableFieldsCSVEmitter'
+require_relative 'twb/analysis/datasources/categoricalColorCodingAnalyzer'
 require_relative 'twb/analysis/datasources/googlesheetdatasourcesanalyzer'
 require_relative 'twb/analysis/datasources/parametersanalyzer'
+require_relative 'twb/analysis/datasources/DataSourceOriginsAnalyzer'
 require_relative 'twb/analysis/Sheets/WorksheetDataStructureCSVEmitter'
 require_relative 'twb/analysis/Sheets/sheetfiltersanalyzer'
 require_relative 'twb/analysis/Sheets/sheetfieldsanalyzer'
 require_relative 'twb/analysis/Sheets/dashsheetsanalyzer'
 
 
-# Represents Tableau Workbooks and their contents.
+# Represents Tableau Workbooks, their contents, and classes that analyze and manipulate them.
 #
 module Twb
-  VERSION = '3.9.7'
+  VERSION = '4.3.1'
 end

data/lib/twb/analysis/CalculatedFields/CalculatedFieldsAnalyzer.rb

@@ -21,7 +21,7 @@ require 'csv'
 module Twb
 module Analysis
 
-  class CalculatedFieldsAnalyzer # < Twb::Util::Graph
+  class CalculatedFieldsAnalyzer
 
       include TabTool
       include Graph
@@ -83,7 +83,7 @@ DOTHEADER
 
       def initialize
           init
-          @funcdoc = {:class=>self.class, :blurb=>'Analyze Calculated Fields from Tableau Workbooks.', :description=>'Calculated fields can be complex, this tool provides robust coverage.',}
+          @funcdoc = {:class=>self.class, :blurb=>'Analyze Calculated Fields', :description=>'Calculated fields can be complex, this tool provides robust coverage.',}
           #-- CSV records collectors
           @csvCalculatedFields = []
           @csvFormulaFields    = []
@@ -101,6 +101,8 @@ DOTHEADER
           #--
           @localEmit  = false
           @imageFiles = []
+          #--
+          @doGraph = config(:dograph)
       end
 
       def processTWB workbook
@@ -164,16 +166,21 @@ DOTHEADER
             calculatedFields      = SortedSet.new
             fieldFormulaLines     = [] 
             referencedFields      = SortedSet.new
-            dataSourceNode = Twb::Util::Graphnode.new(name: ds.uiname, id: ds.id, type: ds, properties: {workbook: @twb.name})
-            @nodes.add dataSourceNode
+            # if @doGraph
+              dataSourceNode = Twb::Util::Graphnode.new(name: ds.uiname, id: ds.id, type: ds, properties: {workbook: @twb.name})
+              @nodes.add dataSourceNode
+            # end
             #-- process Calculatred Fields
             ds.calculatedFields.each do |calcField|
+              emit "Calculated Field: #{calcField}"
               calculatedFields.add calcField.id
               dsFields[calcField.uiname] = calcField
-              calcFieldNode    = Twb::Util::Graphnode.new(name: calcField.uiname, id: calcField.id, type: calcField, properties: {:DataSource => ds.uiname})
-              @nodes.add calcFieldNode
-              dsFieldEdge     = Twb::Util::Graphedge.new(from: dataSourceNode, to: calcFieldNode, relationship: 'contains')
-              @edges.add dsFieldEdge
+              # if @doGraph
+                calcFieldNode    = Twb::Util::Graphnode.new(name: calcField.uiname, id: calcField.id, type: calcField, properties: {:DataSource => ds.uiname})
+                @nodes.add calcFieldNode
+                dsFieldEdge     = Twb::Util::Graphedge.new(from: dataSourceNode, to: calcFieldNode, relationship: 'contains')
+                @edges.add dsFieldEdge
+              # end
               calculation     = calcField.calculation
               if calculation.has_formula
                   #-- collect field formulas as single lines
@@ -201,9 +208,9 @@ DOTHEADER
                                             ]
                   #-- collect individual formula lines
                   flnum = 0
-                  emit "@@ FL: #{calcField.uiname}"
+                  emit "@@ calcField.uiname: #{calcField.uiname}"
                   calculation.formulaResolvedLines.each do |fl|
-                    emit "@@ FL:   => '#{fl}'"
+                    emit "@@ resolved line::   => '#{fl}'"
                     fieldFormulaLines << [ @calculatedFieldsCount,        # 'Calc Field #',
                                            @twb.name,                     # 'Workbook',      
                                            @twbDir,                       # 'Workbook Dir',
@@ -219,24 +226,32 @@ DOTHEADER
                                          ]
                   end
                   #-- collect fields referenced in formula
+                  emit "# Calculated Fields: #{calculation.calcFields.length}"
                   calculation.calcFields.each do |rf|
-                    emit "  rf.name  :'#{rf.name}'"
-                    emit "  rf.uiname:'#{rf.uiname}'"
-                    properties = {'DataSource' => ds.uiname, 'DataSourceReference' => 'local', :source => rf}
-                    refFieldNode  = Twb::Util::Graphnode.new(name: rf.uiname, id: rf.id,   type: rf.type, properties: properties)
-                    @nodes.add refFieldNode
-                    fieldFieldEdge = Twb::Util::Graphedge.new(from: calcFieldNode,  to: refFieldNode, relationship: 'references')
-                    @edges.add fieldFieldEdge
+                    emit "  referenced field       ::'#{rf}'"
+                    emit "  referenced field.name  ::'#{rf.name.nil?}'  :: '#{rf.name}'"
+                    emit "  referenced field.uiname::'#{rf.uiname}'"
+                    # if @doGraph
+                    unless rf.uiname.nil?
+                      properties = {'DataSource' => ds.uiname, 'DataSourceReference' => 'local', :source => rf}
+                      refFieldNode  = Twb::Util::Graphnode.new(name: rf.uiname, id: rf.id,   type: rf.type, properties: properties)
+                      @nodes.add refFieldNode
+                      fieldFieldEdge = Twb::Util::Graphedge.new(from: calcFieldNode,  to: refFieldNode, relationship: 'references')
+                      @edges.add fieldFieldEdge
+                    end
+                    # end
                     referencedFields.add rf.id
                     refFieldTable = ds.fieldTable(rf.name)
                     emit "refFieldTable.nil? : #{refFieldTable.nil?}"
                     unless refFieldTable.nil?
                       tableID        = refFieldTable + ':::' + ds.uiname
                       tableName      = "||#{refFieldTable}||"
-                      tableNode      = Twb::Util::Graphnode.new(name: tableName,      id: tableID,   type: :DBTable, properties: properties)
-                      @nodes.add tableNode
-                      fieldFieldEdge = Twb::Util::Graphedge.new(from: refFieldNode,  to: tableNode, relationship: 'is a field in')
-                      @edges.add fieldFieldEdge
+                      # if @doGraph
+                        tableNode      = Twb::Util::Graphnode.new(name: tableName,      id: tableID,   type: :DBTable, properties: properties)
+                        @nodes.add tableNode
+                        fieldFieldEdge = Twb::Util::Graphedge.new(from: refFieldNode,  to: tableNode, relationship: 'is a field in')
+                        @edges.add fieldFieldEdge
+                      # end
                       # fldToDsNode    = tableNode
                     end
                     @csvFormulaFields <<  [ 
@@ -259,8 +274,10 @@ DOTHEADER
             dsRootFields = calculatedFields - referencedFields
             @referencedFields.merge referencedFields
             @twbRootFields.merge dsRootFields
-            cypher     @twb.name
-            cypherPy   @twb.name
+            if @doGraph
+              cypher     @twb.name
+              cypherPy   @twb.name
+            end
             emit "#######################"
             #-- record calculated fields 
             emit "@@ record calculated fields  ds: #{ds.uiname}"
@@ -333,19 +350,23 @@ DOTHEADER
           end
 
           def cypher twbName
-            cypher = Twb::Util::Cypher.new
-            cypher.fileName = "#{twbName}.calcFields"
-            cypher.nodes = @nodes
-            cypher.edges = @edges
-            cypher.render
+            if @doGraph
+              cypher = Twb::Util::Cypher.new
+              cypher.fileName = "#{twbName}.calcFields"
+              cypher.nodes = @nodes
+              cypher.edges = @edges
+              cypher.render
+            end
           end
 
           def cypherPy twbName
-            cypher = Twb::Util::CypherPython.new
-            cypher.fileName = "#{twbName}.calcFields"
-            cypher.nodes = @nodes
-            cypher.edges = @edges
-            cypher.render
+            if @doGraph
+              cypher = Twb::Util::CypherPython.new
+              cypher.fileName = "#{twbName}.calcFields"
+              cypher.nodes = @nodes
+              cypher.edges = @edges
+              cypher.render
+            end
           end
 
           # def graphEdges twb
@@ -424,7 +445,6 @@ DOTHEADER
             dotFile.puts "  }"
           end
 
-
           def labelTypes dotFile
             fromTos = Set.new
             @edges.each do |edge|

data/lib/twb/analysis/CalculatedFields/MarkdownEmitter.rb

@@ -13,7 +13,7 @@
 #  You should have received a copy of the GNU General Public License
 #  along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
-# require 'twb'
+require 'kramdown'
 
 module Twb
 module Analysis
@@ -26,15 +26,34 @@ module CalculatedFields
 
     def initialize
       init
-      @funcdoc = {:class=>self.class, :blurb=>'Generate Markdown doc for Calculated Fields.', :description=>'Creates a Markdown file documenting the Calculated Fields for an individual Workbook.',}
+      @funcdoc = {:class=>self.class, :blurb=>'Create  Markdown files (one per Workbook) documenting Calculated Fields', :description=>'Create  Markdown files (one per Workbook) documenting Calculated Fields' }
       @metrics = {}
     end
 
+    def kramdownAvailable
+      gem_name, *gem_ver_reqs = 'kramdown', '~> 1.17.0'
+      gdep = Gem::Dependency.new(gem_name, *gem_ver_reqs)
+      # find latest that satisifies
+      found_gspec = gdep.matching_specs.max_by(&:version)
+      # instead of using Gem::Dependency, you can also do:
+      # Gem::Specification.find_all_by_name(gem_name, *gem_ver_reqs)
+          # if found_gspec
+          #   puts "Requirement '#{gdep}' already satisfied by #{found_gspec.name}-#{found_gspec.version}"
+          # else
+          #   puts "Requirement '#{gdep}' not satisfied; could be installing..."
+          #   # reqs_string will be in the format: "> 1.0, < 1.2"
+          #   # reqs_string = gdep.requirements_list.join(', ')
+          #   # multi-arg is safer, to avoid injection attacks
+          #   # system('gem', 'install', gem_name, '-v', reqs_string)
+          # end
+    end
+
     def processTWB twb
       # twb = File.basename(twb)
       @twb = twb #Twb::Workbook.new twb
       @docFileName = './ttdoc/' + @twb.name + '.CalculatedFields.md'
       @docFile     = File.open(@docFileName,'w')
+      addDocFile @docFileName,"Markdown file of Calculated fields for Workbook '#{@twb.name}'"
       @docFile.puts "# #{twb.name}"
       dsNames = @twb.datasourceUINames
       @docFile.puts "#{dsNames.length} Data Sources"
@@ -78,6 +97,9 @@ module CalculatedFields
         end
         @docFile.puts "counted #{cnt} calculated fields\n "
       end  # twb.datasources.each
+      if kramdownAvailable
+        emit 'processing Markdown: kramdownAvailable'
+      end
       finis
     end    # def processTwb twb
 

data/lib/twb/analysis/DataSources/categoricalColorCodingAnalyzer.rb

@@ -0,0 +1,99 @@
+#  categoricalColorCodingAnalyzer.rb Copyright (C) 2018  Chris Gerrard
+#
+#  This program is free software: you can redistribute it and/or modify
+#  it under the terms of the GNU General Public License as published by
+#  the Free Software Foundation, either version 3 of the License, or
+#  (at your option) any later version.  
+#
+#  This program is distributed in the hope that it will be useful,
+#  but WITHOUT ANY WARRANTY; without even the implied warranty of
+#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#  GNU General Public License for more details.
+#
+#  You should have received a copy of the GNU General Public License
+#  along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+require 'twb'
+require 'csv'
+
+module Twb
+module Analysis
+
+  class CategoricalColorCodingAnalyzer
+
+    include TabTool
+
+    attr_accessor :localEmit
+
+    def initialize
+      init
+      @funcdoc    = {:class=>self.class, :blurb=>"Analyze Fields' values categorical color coding from Tableau Workbooks.", :description=>nil,}
+      #--
+      docFileName = docFile('CategoricalColorMappings.csv')
+      $csv  = CSV.open(docFileName,'w')
+      $csv  << ['Workbook', 'Workbook Dir', 'Data Source', 'Field Code', 'Field Tech', 'Field', 'Value', 'Colour']
+      addDocFile docFileName, "Workbooks, Data Sources, Fields, and the Fields' members' categorical color codings."
+      #--
+      @twbCnt    = 0
+      @dscnt     = 0
+      @fieldsCnt = 0
+    end
+
+    def metrics
+      {
+        '# of Workbooks'         => @twbcount,
+        '# of data sources'      => @dscnt,
+        '# of Worksheet Fields'  => @fieldsCnt
+      }
+    end
+
+    def processTWB twb
+      @twb = twb
+      twbName  = twb.name
+      twbDir   = twb.dir
+      emit "   -- #{twbName}"
+      @twbCnt += 1
+      @twbDomainsLoaded = false
+      # <style>
+      #   <style-rule element='mark'>
+      #     <encoding attr='color' field='[none:Calculation_267401288043974656:nk]' type='palette'>
+      #       <map to='#1f77b4'>
+      #         <bucket>&quot;Consistently Meets Expectations&quot;</bucket>
+      twb.datasources.each do |ds|
+          dsName = ds.uiname
+          puts "\t - #{dsName}"
+          coloredFields = ds.node.xpath('.//style/style-rule/encoding[@attr="color"]')
+          # puts "\t   #{coloredFields.length} \t #{coloredFields.class} \t #{coloredFields.nil?}"
+          coloredFields.each do |cf|
+              puts "Attributes: #{cf.attributes}"
+              fieldCode = cf['field']
+              fieldTech = fieldCode.sub(/^\[/,'').sub(/\]$/,'').sub(/^(none|attr|usr):/,'').sub(/:nk$/,'')
+              fieldUI   = ds.fieldUIName fieldTech
+              # puts "\t   - #{field}"
+              maps  = cf.xpath('./map')
+              maps.each do |map|
+                  puts "MAP\n---\n#{map}\n--"
+                  color  = map['to']
+                  value  = ''
+                  values = map.xpath('.//bucket')
+                  unless values.nil?
+                    if values.length == 1
+                      value = values.first.text.gsub(/^"/,'').gsub(/"$/,'')
+                    else
+                      values.each do |bv| 
+                        value += "::#{bv.text}"
+                      end
+                    end
+                  end
+                  # puts "\t     - #{color}  ->  #{value}"
+                  $csv << [twbName, twbDir, dsName, fieldCode, fieldTech, fieldUI, value, color]
+                  puts "VALUE: #{value}"
+              end
+          end
+      end
+    end
+
+  end #class CategoricalColorCodingAnalyzer
+
+end # module Analysis
+end # module Twb

data/lib/twb/analysis/DataSources/dataSourceOriginsAnalyzer.rb

@@ -0,0 +1,71 @@
+#  DataSourceOriginsAnalyzer.rb Copyright (C) 2018  Chris Gerrard
+#
+#  This program is free software: you can redistribute it and/or modify
+#  it under the terms of the GNU General Public License as published by
+#  the Free Software Foundation, either version 3 of the License, or
+#  (at your option) any later version.  
+#
+#  This program is distributed in the hope that it will be useful,
+#  but WITHOUT ANY WARRANTY; without even the implied warranty of
+#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#  GNU General Public License for more details.
+#
+#  You should have received a copy of the GNU General Public License
+#  along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+require 'twb'
+require 'csv'
+
+module Twb
+module Analysis
+
+  class DataSourceOriginsAnalyzer
+
+    include TabTool
+
+    @@recNum = 0
+
+    attr_accessor :localEmit
+
+    def initialize
+      init
+      @funcdoc    = {:class=>self.class, :blurb=>'Analyze Data Source Origins, i.e. where the data comes from', :description=>nil,}
+      #--
+      docFileName = docFile('DataSourceOrigins.csv')
+      @csvFile  = CSV.open(docFileName,'w')
+      @csvFile  << ['Workbook', 'Workbook Version', 'Workbook Directory', 'Data Source', 'Data Source (tech)', 'Is Published?', "Rec #"]
+      addDocFile docFileName, "Workbooks, Data Sources, and the Data Sources' origins"
+      #--
+      @twbCnt     = 0
+      @dsCnt      = 0
+    end
+
+    def metrics
+      {
+        '# of Workbooks'    => @twbcount,
+        '# of Data Sources' => @dsCnt,
+      }
+    end
+
+    def processTWB twb
+       @twb = twb
+       emit "   -- twb:: #{@twb.name}"
+       @twbCnt += 1
+       @twbDomainsLoaded = false
+       parseDataSources
+       finis
+    end
+
+    private
+
+    def parseDataSources
+      @twb.datasources.each do |ds|
+        @csvFile << [@twb.name, @twb.version, @twb.dir, ds.uiname, ds.name, ds.isPublished, @@recNum+=1]
+        @dsCnt += 1
+      end
+    end
+
+  end #class SheetFieldsAnalyzer
+
+end # module Analysis
+end # module Twb

data/lib/twb/analysis/DataSources/dataSourcesLocationsAnalyzer.rb

@@ -0,0 +1,122 @@
+#  sheetfieldsanalyzer.rb Copyright (C) 2018  Chris Gerrard
+#
+#  This program is free software: you can redistribute it and/or modify
+#  it under the terms of the GNU General Public License as published by
+#  the Free Software Foundation, either version 3 of the License, or
+#  (at your option) any later version.  
+#
+#  This program is distributed in the hope that it will be useful,
+#  but WITHOUT ANY WARRANTY; without even the implied warranty of
+#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#  GNU General Public License for more details.
+#
+#  You should have received a copy of the GNU General Public License
+#  along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+require 'twb'
+require 'csv'
+
+module Twb
+module Analysis
+
+  class SheetSourcesAnalyzer
+
+    include TabTool
+
+    attr_accessor :localEmit
+
+    def initialize
+      init
+      @funcdoc    = {:class=>self.class, :blurb=>'Analyze Worksheet Fields', :description=>nil,}
+      #--
+      docFileName = docFile('WorksheetFields.csv')
+      @sheetFieldsCSV  = CSV.open(docFileName,'w')
+      @sheetFieldsCSV  << ['Workbook','Worksheet','Data Source','Data Source (tech)','Field','Field (tech)','Usage']
+      addDocFile docFileName, "Workbooks, Worksheets, and the Sheets' Data Sources and Fields"
+      #--
+      @twbCnt     = 0
+      @sheetCnt   = 0
+      @fieldsCnt  = 0
+    end
+
+    def metrics
+      {
+        '# of Workbooks'         => @twbcount,
+        '# of Worksheets'        => @sheetCnt,
+        '# of Worksheet Fields'  => @fieldsCnt
+      }
+    end
+
+    def processTWB twb
+       @twb = twb
+       emit "   -- twb:: #{@twb.name}"
+       @twbCnt += 1
+       @twbDomainsLoaded = false
+       parseSheets
+       finis
+    end
+
+    private
+
+    def parseSheets
+      @worksheets  = @twb.worksheets
+      @worksheets.each do |sheet|
+        @sheet     = sheet.name
+        @sheetCnt += 1
+        emit "SHEET: #{@sheet}"
+        showFields sheet unless sheet.datasourceFields.nil?
+      end
+    end
+
+    def showFields sheet
+      showDBFields  sheet
+      showRCFields  sheet.rowFields, :row
+      showRCFields  sheet.colFields, :column
+    end
+
+    def showDBFields sheet
+      fields = sheet.datasourceFields
+      emit "def showDBFields sheet: #{sheet.name} #FIELDS: #{fields.length}"
+      if fields.nil?
+        @sheetFieldsCSV  << [@twb.name, @sheet, nil, nil, nil, nil, nil]
+      end
+      fields.each do |dsName, dsfields|
+        ds = @twb.datasource dsName
+        emit "      ds: #{dsName}"
+        emit "        - #{ds.uiname}"
+        emit "        : #{ds.class}"
+        dsfields.each do |sheetField|
+           @fieldsCnt += 1
+           emit "       f: #{sheetField}"
+           emit "       c: #{sheetField.class}"
+           fuiName = ds.fieldUIName sheetField #Fields[sheetField]
+           @sheetFieldsCSV  << [@twb.name, @sheet, ds.uiname, dsName, sheetField.uiname, sheetField.name, 'DB ref']
+           # emit true, "        : #{dsFields[field].class}"
+        end
+      end
+    end
+
+    def showRCFields fields, usage
+      emit "def showRCFields #fields: #{fields.length}  \t #{fields}"
+      if fields.nil?
+        @sheetFieldsCSV  << [@twb.name, @sheet, nil, nil, nil, nil, nil]
+      else
+        fields.each do |cf|
+          emit "coded field: #{cf}"
+          fldName = cf.name
+          dsName  = cf.dataSource
+          ds      = @twb.datasource cf.dataSource
+          emit "DATASOURCE : #{ds.class} "  #{ }" #{ds}"
+          fuiName = ds.fieldUIName cf.name
+          emit "      ds: #{dsName}"
+          emit "        - #{ds.uiname}"
+          emit "        : #{ds.class}"
+          @sheetFieldsCSV  << [@twb.name, @sheet, ds.uiname, dsName, fuiName, fldName, usage]
+        end
+      end
+    end
+
+  end #class SheetFieldsAnalyzer
+
+end # module Analysis
+end # module Twb