twb 4.9.3 → 5.2.0

data/lib/twb/analysis/calculatedfields/dotanalyzer.rb

@@ -0,0 +1,139 @@
+#  dotanalyzer.rb - this Ruby script Copyright 2017, 2018 Christopher Gerrard
+#
+#  This program is free software: you can redistribute it and/or modify
+#  it under the terms of the GNU General Public License as published by
+#  the Free Software Foundation, either version 3 of the License, or
+#  (at your option) any later version.
+#
+#  This program is distributed in the hope that it will be useful,
+#  but WITHOUT ANY WARRANTY; without even the implied warranty of
+#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#  GNU General Public License for more details.
+#
+#  You should have received a copy of the GNU General Public License
+#  along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+
+module Twb
+module Analysis
+module CalculatedFields
+
+  class DotAnalyzer
+    include TabTool
+
+    attr_reader :docFileName
+
+    @@gvDotLocation = 'C:\\tech\\graphviz\\Graphviz2.38\\bin\\dot.exe'
+    @@imageTypes = ['pdf', 'png', 'svg']
+
+    def initialize(**args)
+      @args = args
+      init
+      @funcdoc = {:class=>self.class, :blurb=>'Create Dot files documenting Calculated Fields', :description=>'Analyze Calculated Fields - create Dot files' }
+      @metrics = {}
+      @imageFiles = Array.new
+    end
+
+    def processTWB twb
+      # twb = File.basename(twb)
+      @twb         = twb #Twb::Workbook.new twb
+      addDocFile @dotFile, @dotFileName, "Dot file of Calculated fields for Workbook '#{@twb.name}'"
+      @twb.datasources.each do |ds|
+        unless ds.calculatedFields.empty?
+          initDotFile ds.uiname
+          # @dotFile.puts "\n ## #{ds.uiname} \n "
+          # @dotFile.puts "__has #{ds.calculatedFields.length} calculated fields__\n "
+          @cfCnt = 0
+          calcFields = Set.new
+          refFields  = Set.new
+          edges      = Set.new
+          ds.calculatedFields.each do |cf|
+            @cfCnt += 1
+            calcFields << cf.uiname
+            edges            << " \"#{ds.uiname}\" -> \"#{cf.uiname}\"  [tailport=e, headport=w] "
+            cf.referencedFields.each do |rf|
+              refFields << rf.uiname
+              edges            << " \"#{cf.uiname}\" -> \"#{rf.uiname}\" [tailport=e, headport=w] "
+            end
+          end # ds.calculatedFields.each
+          # "federated.17h7owt0rsacke17cql8o0w2ittk"  ->  "New AO Actuals Query in PP+ (AO Variance Data)::vs Prior Year [YTD]"
+          # "federated.01s5lca037ted31gxs9sg0t9mnnt" [label="Controls"    ]
+          edges.each do |edge|
+            @dotFile.puts "\t #{edge.strip}"
+          end
+          @dotFile.puts " "
+          allFields = calcFields + refFields
+          allFields.each do |f|
+            @dotFile.puts "\t \"#{f}\" [label=\"#{f}\"]"
+          end
+          endPointFields = allFields - calcFields
+          rankSame(endPointFields) unless endPointFields.nil? || endPointFields.empty?
+          closeDotFile
+          @@imageTypes.each do |type|
+            renderDot type
+          end
+        end
+      end  # twb.datasources.each
+      finis
+    end    # def processTwb twb
+
+    private
+
+    def initDotFile dsName
+      @dotFileName = './ttdoc/' + @twb.name + '.' + dsName + '.CalculatedFields.dot'
+      @dotFile     = File.open(@dotFileName,'w')
+      # @dotFile.puts @@dotHeader
+      @dotFile.puts ' digraph g {'
+      @dotFile.puts '   graph [rankdir="LR" splines=line];'
+      @dotFile.puts '   node  [shape="box"  width="2"];'
+      @dotFile.puts ' '
+      @dotFile.puts '   subgraph cluster_0 {'
+    end
+
+    def closeDotFile
+      # @dotFile.puts "\n # counted #{@cfCnt} calculated fields\n "
+      # @dotFile.puts "\n }"
+     @dotFile.puts '   }'
+      @dotFile.puts ' '
+      @dotFile.puts '// -------------------------------------------------------------'
+     @dotFile.puts ' '
+     @dotFile.puts '   subgraph cluster_1 {'
+      #@dotFile.puts '     color=white;'
+     @dotFile.puts '     style=invis;'
+      #@dotFile.puts '     border=0;'
+     @dotFile.puts '     node [border=blue];'
+     @dotFile.puts ' '
+     @dotFile.puts '     "" [style=invis]'
+     @dotFile.puts "     \"Tableau Tools\\nCalculated Fields Map\\nWorkbook '#{@twb.name}'\\n#{Time.new.ctime}\"  [penwidth=0]"
+      #@dotFile.puts "     \"Tableau Tools Workbook  Calculated Fields Map\\n#{Time.new.ctime}\"  -> \"\"  [style=invis]"
+     @dotFile.puts ' '
+     @dotFile.puts '   }'
+     @dotFile.puts ' '
+     @dotFile.puts '}'
+      @dotFile.close
+    end
+
+      def rankSame fields
+        @dotFile.puts "\n {rank=same "
+        fields.each do |f|
+          @dotFile.puts "\t \"#{f}\" "
+        end
+        @dotFile.puts " } "
+      end
+
+      def renderDot format
+        imageType = '-T' + format
+        imageFile  = @dotFileName + '.Graph.' + format
+        imageParam = '-o"' + imageFile + '"'
+        emit "system  #{@@gvDotLocation} #{imageType} #{imageParam} \"#{@dotFileName}\""
+        system "#{@@gvDotLocation} #{imageType} #{imageParam} \"#{@dotFileName}\""
+        # emit  "           - #{imageFile}"
+        @imageFiles << imageFile
+        return imageFile
+      end
+
+  end # class DotAnalyzer
+
+end   # nodule CalculatedFields
+end   # module Analysis
+end   # module Twb

data/lib/twb/analysis/calculatedfields/markdownemitter.rb

@@ -52,10 +52,10 @@ module CalculatedFields
             @docFile.puts "#{l.gsub('<<','[').gsub('>>',']')}" 
           end
           @docFile.puts "```"
-          if cf.calcFields.length > 0
+          if cf.referencedFields.length > 0
             fieldsRefOrder  = []
             fieldsSortSet   = SortedSet.new
-            cf.calcFields.each do |field|
+            cf.referencedFields.each do |field|
               fieldsRefOrder.push field.uiname
               fieldsSortSet << field.uiname
             end

data/lib/twb/analysis/calculatedfields/t.rb

@@ -0,0 +1,6 @@
+require 'twb'
+
+twb = Twb::Workbook.new 'MS.twb'
+ds  = twb.datasources.to_a.last
+cfs =  ds.calculatedFields
+cfs.each { |cf|  puts "\n\n--\n#{cf.calculation.formula}\n--"; cf.calcFields.each { |f| puts "   %-12s %s " % [f[:ds], f[:field] ] } }

data/lib/twb/analysis/datasources/datasourcefilesanalyzer.rb

@@ -14,6 +14,7 @@
 #  along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
 require 'csv'
+require 'pry'
 
 module Twb
 module Analysis
@@ -32,6 +33,7 @@ module DataSources
                           'Data Source', 
                           'File Name', 
                           'File Dir',
+                          'File Exists?',
                           'File Modified',
                           'File Size'
                         ]
@@ -55,7 +57,7 @@ module DataSources
 
       def processTWB twb
         @twbName = twb.name
-        @twDir   = twb.dir
+        @twbDir  = twb.dir
         twb.datasources.each do |ds|
           processFiles ds
         end
@@ -69,9 +71,10 @@ module DataSources
             fqFileName  = twbFileDir.nil? ?  twbFileName : twbFileDir + '/' + twbFileName
             fileName = File.basename fqFileName
             fileDir  = File.dirname  fqFileName
-            modtime  = File.mtime    fqFileName
-            size     = File.size     fqFileName
-            data = [$recNum+=1, twbName, twb.dir, ds.uiname, fileName, fileDir, modtime, size]
+            exists   = File.exist?   fqFileName
+            modtime  = exists ? File.mtime(fqFileName) : nil
+            size     = exists ? File.size(fqFileName)  : nil
+            data = [@recNum+=1, @twbName, @twbDir, ds.uiname, fileName, fileDir, exists, modtime, size]
             @csvFile  << data
         end
       end

data/lib/twb/analysis/sheets/dashboardsummarizer.rb

@@ -0,0 +1,94 @@
+#  sheetfieldsanalyzer.rb Copyright (C) 2018  Chris Gerrard
+#
+#  This program is free software: you can redistribute it and/or modify
+#  it under the terms of the GNU General Public License as published by
+#  the Free Software Foundation, either version 3 of the License, or
+#  (at your option) any later version.  
+#
+#  This program is distributed in the hope that it will be useful,
+#  but WITHOUT ANY WARRANTY; without even the implied warranty of
+#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#  GNU General Public License for more details.
+#
+#  You should have received a copy of the GNU General Public License
+#  along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+require 'twb'
+require 'csv'
+
+module Twb
+module Analysis
+
+  class DashboardsSummarizer
+
+    include TabTool
+
+    attr_accessor :localEmit
+
+    def initialize(**args)
+      @args      = args
+      @recordDir = !@args.nil? && @args[:recordDir] == true
+      @ttdocdir  = @args[:ttdocdir]
+      @csvAdd    = !@args.nil? && args[:csvMode] == :add
+      @csvMode   = @csvAdd ? 'a' : 'w'
+      init
+      @funcdoc   = {:class=>self.class, :blurb=>'Analyze Dashboard Worksheets', :description=>'Identifies the Worksheets present in Dashboards.',}
+      #--
+      docFileName     = docFile('DashboardSummaries.csv')
+      @dashboardsCSV  = CSV.open(docFileName,@csvMode)
+      unless @csvAdd
+        if @recordDir
+          @dashboardsCSV  << ['Rec #','Workbook','Dashboard','# Worksheets','Workbook Dir']
+        else
+          @dashboardsCSV  << ['Rec #','Workbook','Dashboard','# Worksheets'               ]
+        end
+      end
+      addDocFile @dashboardsCSV, docFileName, "Workbooks and their Dashboards' summaries"
+      #--
+      @twbCount    = 0
+      @dashCount   = 0
+      @recNum      = 0
+    end
+
+    def metrics
+      {
+        # '# of Workbooks'  => @twbCount,
+        '# of Dashboards' => @dashCount,
+      }
+    end
+
+    def processTWB twb
+       @twb     = twb
+       @twbName = @twb.name
+       @twbDir  = @twb.dir
+       @modTime = @twb.modtime
+       emit "   -- #{@twbName}"
+       @twbCount += 1
+       parseDashes
+       finis
+    end
+
+    def parseDashes
+      @dashboards = @twb.dashboards
+      @dashboards.each do |dash|
+        emit "DASH:: #{dash.name}"
+        @dashCount += 1
+        recordCSV [@twbName, dash.name, dash.worksheets.length]
+      end
+    end
+
+    private
+
+      def recordCSV record
+        numberedRec = [@recNum+=1] + record
+        if @recordDir
+          @dashboardsCSV  << numberedRec.push(@twbDir)
+        else
+          @dashboardsCSV  << numberedRec
+        end
+      end
+
+  end #class SheetFieldsAnalyzer
+
+end # module Analysis
+end # module Twb

data/lib/twb/analysis/workbooksummaryanalyzer.rb

@@ -37,7 +37,7 @@ module Analysis
       docFileName  = docFile('WorkbookSummary.csv')
       @csvFile     = CSV.open(docFileName,@csvMode)
       unless @csvAdd
-        csvHeader       = ['Rec #', 'Workbook','Type','Version','Build','Platform','Modified','# Data Sources','# Dashboards','# Worksheets']
+        csvHeader       = ['Rec #', 'Workbook','Directory','Type','Version','Build','Platform','Modified','# Data Sources','# Dashboards','# Worksheets']
         if @recordDir
           csvHeader.push('Workbook Directory')
         end
@@ -65,6 +65,7 @@ module Analysis
        @twb     = twb
        emit "   -- #{@twbName}"
        recordCSV [ @twb.name,
+                   @twb.dir, 
                    @twb.type, 
                    @twb.version, 
                    @twb.build,

data/lib/twb/calculatedfield.rb

@@ -1,4 +1,4 @@
-#  Copyright (C) 2014, 2015, 2017  Chris Gerrard
+#  Copyright (C) 2014, 2015, 2020 Chris Gerrard
 #
 #  This program is free software: you can redistribute it and/or modify
 #  it under the terms of the GNU General Public License as published by
@@ -26,7 +26,7 @@ module Twb
     attr_reader :node,        :properties
     attr_reader :caption,     :name,       :uiname
     attr_reader :datatype,    :role,       :propType  
-    attr_reader :calculation, :calcFields 
+    attr_reader :calculation, :referencedFields 
     attr_reader :isGroup,     :groupMembers
     attr_reader :hidden
 
@@ -52,8 +52,8 @@ module Twb
       @properties ||= loadProperties
     end
 
-    def calcFields
-        @calculation.calcFields
+    def referencedFields
+        @calculation.referencedFields
     end
 
     def formulaLines

data/lib/twb/columnfield.rb

@@ -14,6 +14,7 @@
 #  along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
 require 'nokogiri'
+require 'pry'
 
 module Twb
 
@@ -85,7 +86,6 @@ module Twb
       @aggregation        = load 'aggregation'
       @autoColumn         = load 'auto-column'
       @datatypeCustomized = load 'datatype-customized'
-      # @calcField          = loadCalcField
     end
 
     def id
@@ -94,7 +94,7 @@ module Twb
 
     def load nodeName
       attr = @node.attribute(nodeName)
-      val  = attr.nil? ? nil : attr.text.strip.gsub(/^\[|\]$/,'')
+      val  = attr.nil? ? nil : attr.text.strip
       return val
     end
 

data/lib/twb/datasource.rb

@@ -56,7 +56,7 @@ module Twb
     attr_reader :tableFieldsMap
     attr_reader :fieldUINames
     attr_reader :aliases
-    attr_reader :calculatedFields, :calculatedFieldNamesMap, :calculatedFieldNames, :calculatedField 
+    attr_reader :calculatedFields, :calculatedFieldsMap, :calculatedFieldNames, :calculatedField 
     attr_reader :allFields 
     attr_reader :groups 
     attr_reader :filters
@@ -391,40 +391,6 @@ module Twb
       return @fieldUINames
     end
 
-    def loadFieldUINames
-      # puts 'loadFieldUINames'
-      @fieldUINames = {}
-      # puts 'metadataFields'
-      metadataFields.each do |fld|
-        # puts " - name:%-45s   |  uiname:%-45s   |  localName:%-45s " % [ "'#{fld.name}'", "'#{fld.uiname}'", "'#{fld.localName}'"]
-        @fieldUINames[fld.uiname]    = fld.uiname
-        @fieldUINames[fld.localName] = fld.uiname unless  fld.localName.nil?
-        @fieldUINames[fld.name]      = fld.uiname unless  fld.name.nil?
-      end
-      # puts 'calculatedFields'
-      calculatedFields.each do |fld| 
-        # puts " - name:%-45s   |  uiname:%-45s " % [ "'#{fld.name}'", "'#{fld.uiname}'"]
-        @fieldUINames[fld.name]   = fld.uiname
-        @fieldUINames[fld.uiname] = fld.uiname
-      end
-      # puts 'localFields'
-      localFields.each do |fld|
-        # puts " - name:%-45s   |  uiname:%-45s " % [ "'#{fld.name}'", "'#{fld.uiname}'"]
-        @fieldUINames[fld.name]   = fld.uiname
-        @fieldUINames[fld.uiname] = fld.uiname
-      end
-      # puts "columnFields: #{columnFields.length}"
-      columnFields.each do |fld|
-        @fieldUINames[fld.name]   = fld.uiname
-        @fieldUINames[fld.uiname] = fld.uiname
-      end
-      groups.each do |fld|
-        @fieldUINames[fld.name]   = fld.uiname
-        @fieldUINames[fld.uiname] = fld.uiname
-      end
-      return @fieldUINames
-    end
-
     def tableFields
       @tableFieldsMap.values
     end
@@ -482,8 +448,7 @@ module Twb
 
     def loadAllFields
       @allFields = SortedSet.new
-      dbf = dbFields
-      @allFields << dbf
+      @allFields << dbFields
       @allFields << calculatedFieldNames
     end
 
@@ -532,6 +497,53 @@ module Twb
 
     private
 
+    def loadFieldUINames
+      @fieldUINames = {}
+       # puts"metadataFields: #{metadataFields.length}"
+      metadataFields.each do |fld|
+         # puts" - name:%-45s   |  caption:%-45s   |  uiname:%-45s " % [ "'#{fld.name}'",  "'#{fld.caption}'", "'#{fld.uiname}'"]
+        unless fld.name.nil?
+          @fieldUINames[fld.uiname]    = fld.uiname
+          @fieldUINames[fld.localName] = fld.uiname unless  fld.localName.nil?
+          @fieldUINames[fld.name]      = fld.uiname
+        end
+      end
+       # puts"localFields: #{localFields.length}"
+      localFields.each do |fld|
+         # puts" - name:%-45s   |  caption:%-45s   |  uiname:%-45s " % [ "'#{fld.name}'",  "'#{fld.caption}'", "'#{fld.uiname}'"]
+        unless fld.caption.nil?
+          @fieldUINames[fld.name]   = fld.caption
+          @fieldUINames[fld.uiname] = fld.caption
+        end
+      end
+       # puts"groups: #{groups.length}"
+      groups.each do |fld|
+         # puts" - name:%-45s   |  caption:%-45s   |  uiname:%-45s " % [ "'#{fld.name}'",  "'#{fld.caption}'", "'#{fld.uiname}'"]
+        unless fld.caption.nil?
+          @fieldUINames[fld.name]   = fld.caption
+          @fieldUINames[fld.uiname] = fld.caption
+        end
+      end
+       # puts"calculatedFields: #{calculatedFields.length}"
+      calculatedFields.each do |fld| 
+         # puts" - name:%-45s   |  caption:%-45s   |  uiname:%-45s " % [ "'#{fld.name}'",  "'#{fld.caption}'", "'#{fld.uiname}'"]
+        unless fld.caption.nil?
+          @fieldUINames[fld.name]   = fld.caption
+          @fieldUINames[fld.uiname] = fld.caption
+        end
+      end
+       # puts"columnFields: #{columnFields.length}"
+      columnFields.each do |fld|
+         # puts" - name:%-45s   |  caption:%-45s   |  uiname:%-45s " % [ "'#{fld.name}'",  "'#{fld.caption}'", "'#{fld.uiname}'"]
+        unless fld.caption.nil?
+          # @fieldUINames[fld.name]                          = fld.caption
+          @fieldUINames[fld.name.gsub(/^[\[]|[\]]$/,'')]   = fld.caption
+          @fieldUINames[fld.uiname]                        = fld.caption
+        end
+      end
+      return @fieldUINames
+    end
+
     def loadGroups
       @groups = []
       groupNodes = @node.xpath('.//group')