twb 1.0.5 → 1.9.1

data/lib/twb/analysis/calculatedfieldsanalyzer.rb

@@ -1,508 +0,0 @@
-#  calculatedfieldsanalyzer.rb - this Ruby script Copyright 2017 Christopher Gerrard
-#
-#  This program is free software: you can redistribute it and/or modify
-#  it under the terms of the GNU General Public License as published by
-#  the Free Software Foundation, either version 3 of the License, or
-#  (at your option) any later version.
-#
-#  This program is distributed in the hope that it will be useful,
-#  but WITHOUT ANY WARRANTY; without even the implied warranty of
-#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-#  GNU General Public License for more details.
-#
-#  You should have received a copy of the GNU General Public License
-#  along with this program.  If not, see <http://www.gnu.org/licenses/>.
-
-require 'nokogiri'
-require 'twb'
-require 'set'
-require 'csv'
-require 'logger'
-
-module Twb
-module Analysis
-  
-  class CalculatedFieldsAnalyzer
-
-      attr_reader :calculatedFieldsCount, :formulaFieldsCount
-
-      @@ttlogfile     = 'CalculatedFieldsAnalyzer.ttlog'
-      @@gvDotLocation = 'C:\\tech\\graphviz\\Graphviz2.38\\bin\\dot.exe'
-      @@processName   = '.CalculatedFields'
-
-      @@calcFieldsCSVFileName   = 'TwbCalculatedFields.csv'
-      @@calcFieldsCSVFileHeader = ['Record #',
-                                   'Workbook',      'Workbook Dir',
-                                   'Data Source',   'Data Source Caption', 'Data Source Name (tech)',
-                                   'Field Name',    'Field Caption',       'Field Name (tech)',
-                                   'Data Source + Field Name (tech)',
-                                   'Data Type',     'Role',  'Type',       
-                                   'Class',
-                                   'Scope Isolation',
-                                   'Formula Length',
-                                   'Formula Code',
-                                   'Formula',
-                                   'Formula Comments',
-                                   'Formula LOD?'
-                                  ]
-
-      @@formFieldsCSVFileName   = 'TwbFormulaFields.csv'
-      @@formFieldsCSVFileHeader = ['Rec #',
-                                   'Workbook',   'Workbook Dir',
-                                   'Data Source',
-                                   'Field - Calculated',
-                                   'Data Source - Formula (tech)', 
-                                   'Data Source - Formula', 
-                                   'Field - Formula (tech)',
-                                   'Field - Formula',
-                                   'Data Source + Field - Calculated',
-                                   'Table'
-                                ]
-
-      @techUINames = {}
-      @fieldTables = {}
-
-
-      @@dotHeader = <<DOTHEADER
-  digraph g {
-      graph [rankdir="LR" splines=line];
-      node  [shape="box"  width="2"];
-
-DOTHEADER
-
-      def initialize 
-          #-- Logging setup --
-            @logger = Logger.new(@@ttlogfile)
-            @logger.level = Logger::DEBUG
-          #-- CSV files setup --
-            @calcFieldsCSVFile = CSV.open(@@calcFieldsCSVFileName,'w')
-            @calcFieldsCSVFile <<         @@calcFieldsCSVFileHeader
-            # --
-            @formFieldsCSVFile = CSV.open(@@formFieldsCSVFileName ,'w')
-            @formFieldsCSVFile <<         @@formFieldsCSVFileHeader
-          #-- Counters setup --
-            @twbCount              = 0
-            @calculatedFieldsCount = 0
-            @formulaFieldsCount    = 0
-          # --
-            @referencedFields     = SortedSet.new
-          # --
-            @localEmit  = false
-            emit  "\n\nLogging activity to: #{File.basename(@@ttlogfile)}"
-            @imageFiles  = []
-      end
-
-      def loadFieldTables dataSource
-             emit  "FIELD TABLES"
-        @records = CSV.read('C:\Professional\Clients\Incapsulate\Internal Project Monitoring\Project Portfolio v2\Salesforce Fields.csv')
-        @records.each do |rec|
-             emit  "-- #{rec}"
-          m = {}
-          m['table'] = rec[1]
-          m['dbFieldName'] = rec[2]
-          @fieldTables[rec[0]] = m
-        end
-             emit  "=========="
-             emit  @fieldTables
-             emit  "=========="
-             emit  "FIELD TABLES"
-      end
-
-      def processTWB twbWithDir
-        twb = File.basename(twbWithDir)
-        @twb = Twb::Workbook.new twbWithDir
-              emit  "- Workbook: #{twbWithDir}"
-              emit  "   version: #{@twb.version}"
-        return if twbWithDir.end_with? == "Tableau Calculated Fields Analyses.twb"
-        twbDir  = File.dirname(File.expand_path(twbWithDir))
-        edges   = Set.new
-        # -- processing
-        dss = @twb.datasources
-        twbRootFields = Set.new
-        dss.each do |ds|
-          emit  "Datasource: '#{ds.uiname}'    -> #{ds.Parameters?}"
-          next if ds.Parameters?  # don't process the Parameters data source
-                                  # it requires special handling, has different XML structure
-          #-- For tracking unreferenced (root) calculated fields = calculatedFields - referencedFields
-          calculatedFields = SortedSet.new
-          referencedFields = SortedSet.new
-          #-- 
-          dsTechName  = ds.name
-          dsCaption   = ds.caption
-          dsName      = ds.uiname
-          dsID        = dsTechName + ':::' + dsName
-          emit  "\n\n "
-          emit  "======================================================"
-          emit  "======================================================"
-          emit   "=======    DATA SOURCE: #{ds.uiname}    ====== "
-          emit  "======================================================"
-          emit  "======================================================\n\n "
-          dsGraphNode = Twb::Util::Graphnode.new(name: dsName, id: dsID, type: :TwbDataConnection, properties: {workbook: twbWithDir})
-          emit  "\t      dsgnode: #{dsGraphNode}"
-          fieldUINames     = ds.fieldUINames
-          calculationNodes = ds.calculatedFields
-          emit  "calculationNodes  : nil? '#{calculationNodes.nil?}'" # - len '#{calculationNodes.length}'"
-          calculationNodes.each do |calcNode|
-            calculation     = Twb::FieldCalculation.new(calcNode, ds)
-            emit  "HANDLING CALCULATION NODE:"
-            emit  calcNode.attributes
-            #-- field names     --
-            fldCaption,     = calcNode.xpath('../@caption').text
-            fldTechName     = calcNode.xpath('../@name').text.gsub(/^\[/,'').gsub(/\]$/,'')
-            fldName         = if   fldCaption == ''
-                              then fldTechName
-                              else fldCaption
-                              end
-            emit    "\t Field   : #{fldName}"
-            emit    "\t Formula : #{calcNode.attribute('formula')}"
-            dataType        = calcNode.xpath('../@datatype').text
-            role            = calcNode.xpath('../@role').text
-            type            = calcNode.xpath('../@type').text
-            fieldID         = fldTechName+'::'+dsName
-            calculatedFields.add fieldID
-            srcGraphNode    = Twb::Util::Graphnode.new(name: fldName,     id: fieldID,      type: :CalculatedField, properties: {:DataSource => dsName})
-            dsFieldEdge     = Twb::Util::Graphedge.new(from: dsGraphNode, to: srcGraphNode, relationship: 'contains')
-            edges.add dsFieldEdge
-            hasFormula      = calcNode.has_attribute?('formula')
-            if hasFormula
-                formulaText  = calcNode.attribute('formula').text
-                emit  "\t     Formula: #{formulaText}"
-                #-- field attributes     --
-                # fldDispLabel    = "#{fldName}\n--\n#{formulaText.gsub('"', "'")}" #fldName + '\n--' + formulaText.to_s
-                emit  "\t     srfnode: #{srcGraphNode} "
-                emit  "\t     dsfedge: #{dsFieldEdge} "
-                emit  " "
-                emit  "\tFIELD    cap: #{fldCaption} "
-                emit  "\t        name: #{fldTechName} "
-                emit  "\t      uiname: #{fldName} "
-                emit  "\t------------------------------------------------------------"
-                #-- calculation     --
-                formulaFlat     = formulaText.gsub(/\r\n/, ' ## ').gsub(/\n/, ' ## ').gsub(/[ ]+/,' ')
-                formulaFlatFlat = formulaFlat.upcase
-                formulaLOD      = formulaFlatFlat.include?('{FIXED') || formulaFlatFlat.include?('{INCLUDE') || formulaFlatFlat.include?('{EXCLUDE')
-                formulaLength   = formulaText.length
-                         emit  "\tFORMULA TEXT: #{formulaText} "
-                         emit  "\t        FLAT: #{formulaFlat}"
-                         emit  "\t------------------------------------------------------------"
-                comments        = calculation.comments             # getComments( formulaText )
-                calcClass       = calculation.class                # d.xpath('./@class').text
-                scopeIsolation  = calcNode.xpath('./@scope-isolation').text
-                # -- resolved fields: {internal field name => datasource}
-                # -- datasource is only present for fields located in other data sources
-                resolvedFields  = calculation.resolvedFields
-                # prepare UI formula, replacing technical field names with their UI forms
-                uiFormula = formulaFlat.gsub(' XX ',' ')
-                resolvedFields.each do |rf|
-                  emit  "\tRESOLVED FLD: #{rf.inspect}"
-                  calcFieldName   = rf[:field]
-                  if rf[:source].nil?
-                    calcFieldRef = "[%s]" % [ calcFieldName ]
-                    dispFieldRef = "[%s]" % [ ds.fieldUIName(calcFieldName) ]
-                  else
-                    remoteDS     = @twb.datasource(rf[:source])
-                    remoteDSName = remoteDS.uiname
-                    remoteDSFld  = remoteDS.fieldUIName(calcFieldName)
-                    calcFieldRef = "[%s].[%s]" % [ rf[:source],  calcFieldName ]
-                    dispFieldRef = "[%s].[%s]" % [ remoteDSName, remoteDSFld   ]
-                  end
-                  emit  "\tcalcFieldRef: #{calcFieldRef}"
-                  emit  "\tdispFieldRef: #{dispFieldRef}"
-                  uiFormula    = uiFormula.gsub(calcFieldRef, dispFieldRef)
-                end
-                emit  "\t        FLAT: #{formulaFlat}"
-                emit  "\t    Resolved: #{uiFormula}\n\t--"
-                @calcFieldsCSVFile << [
-                                        @calculatedFieldsCount += 1,
-                                        twb,           twbDir,
-                                        dsName,        dsCaption,      dsTechName,
-                                        fldName,       fldCaption,     fldTechName, 
-                                        dsTechName + '::' + fldTechName,
-                                        dataType,      role,           type, 
-                                        calcClass,     scopeIsolation,
-                                        formulaLength, formulaFlat,    uiFormula,
-                                        comments,
-                                        formulaLOD
-                                       ]
-                resolvedFields.each do |rf|
-                 emit  "\t\t        res field : #{rf[:field]}    "
-                 emit  "\t\t        res source: #{rf[:source]}"
-                  calcFieldName   = rf[:field]
-                  calcDataSource  = rf[:source]
-                  localDataSource = rf[:source].nil?   # if there isn't a rf[:source] value 
-                                                       # the field is from this data source
-                                                       # else the field is from an alien data source (in the same workbook)
-                  refDataSource  = localDataSource ? ds : @twb.datasource(calcDataSource)
-                  dispFieldName  = refDataSource.fieldUIName(calcFieldName)
-                  calcFieldTable = refDataSource.fieldTable(calcFieldName)
-                         emit  "\t\t       calc field : #{dispFieldName}          nil?<#{dispFieldName.nil?}>"
-                         emit  "\t\t       data source: #{refDataSource.uiname}"
-                         emit  "\t\t             table: #{calcFieldTable}         nil?<#{calcFieldTable.nil?}>"
-                  properties = {'DataSource' => dsName, 'DataSourceReference' => 'local'}
-                  if dispFieldName.nil?
-                     dispFieldName = "<#{calcFieldName}>::<#{calcDataSource}> UNDEFINED"
-                     properties['status'] = 'UNDEFINED'
-                  end
-                  calcFieldID = "#{calcFieldName}::#{refDataSource.uiname}"
-                  if !localDataSource
-                    calcFieldID = "#{calcFieldName}:LDS:#{ds.uiname}:RDS:#{refDataSource.uiname}"
-                    properties['DataSourceReference'] = 'remote'
-                  end
-                  calcFieldTable = ds.fieldTable(calcFieldName)
-                  calcFieldType  = calcFieldTable.nil? ? :CalculatedField : :DatabaseField
-                  calcFieldNode  = Twb::Util::Graphnode.new(name: dispFieldName, id: calcFieldID,   type: calcFieldType, properties: properties)
-                  fieldFieldEdge = Twb::Util::Graphedge.new(from: srcGraphNode,  to: calcFieldNode, relationship: 'references')
-                  edges.add fieldFieldEdge
-                  referencedFields.add calcFieldID
-                  # @formulaFieldsCount+=1
-                         emit  "\t\t    calcFieldNode: #{calcFieldNode}"
-                         emit  "\t\t        graphEdge: #{fieldFieldEdge}"
-                  fldToDsNode = calcFieldNode
-                  if !calcFieldTable.nil?
-                     tableID        = calcFieldTable + ':::' + ds.uiname
-                     tableName      = "-[#{calcFieldTable}]-"
-                     tableNode      = Twb::Util::Graphnode.new(name: tableName,      id: tableID,   type: :DBTable, properties: properties)
-                     fieldFieldEdge = Twb::Util::Graphedge.new(from: calcFieldNode,  to: tableNode, relationship: 'is a field in')
-                     edges.add fieldFieldEdge
-                     fldToDsNode    = tableNode
-                  end
-                  if !localDataSource
-                    alienDSNode = Twb::Util::Graphnode.new(       name: '==>' + refDataSource.uiname, 
-                                                                    id: "#{ds.uiname}::::=>#{refDataSource.uiname}", 
-                                                                  type: :DBTable, 
-                                                            properties: {'Home Source' => dsName, 'Remote Source' => refDataSource.uiname}
-                                                          )
-                    fieldFieldEdge  = Twb::Util::Graphedge.new(from: fldToDsNode,  to: alienDSNode, relationship: 'In Remote Data Source')
-                    edges.add fieldFieldEdge
-                  end
-                  @formFieldsCSVFile << [ @formulaFieldsCount+=1, 
-                                          twb,     
-                                          twbDir, 
-                                          dsName,        
-                                          fldName, 
-                                          refDataSource.name,
-                                          refDataSource.uiname,
-                                          calcFieldName, 
-                                          dispFieldName,
-                                          dsName + '::' + dispFieldName,
-                                          'fieldTable'
-                                        ] 
-                end
-            end  # if hasFormula
-          end # calculationNodes.each
-          dsRootFields = calculatedFields - referencedFields
-          @referencedFields.merge referencedFields
-          #--
-          emit "--\nCalculated Fields\n-----------------"
-          calculatedFields.each { |f| emit f }
-          emit "--\nReferenced Fields\n-----------------"
-          referencedFields.each { |f| emit f }
-          emit "--\nDS Root Fields\n-----------------"
-          dsRootFields.each { |f| emit f }
-          emit "--"
-          # --
-          twbRootFields.merge dsRootFields
-        end # dss.each
-        @twbCount += 1
-        mapTwb     twb, edges, twbRootFields
-        graphEdges twb, edges
-        emit  "#######################"
-        return @imageFiles
-      end
-
-
-      def mapTwb twb, edges, rootFields
-        dotFile     = initDot twb
-        dotFileName = File.basename dotFile
-        dotFile.puts "\n //  subgraph cluster_1 {"
-        dotFile.puts "   //    color= grey;"
-        dotFile.puts ""
-        edgesAsStrings = SortedSet.new
-        # this two step process coalesces the edges into a unique set, avoiding duplicating the dot 
-        # file entries, and can be shrunk when graph edges expose the bits necessary for management by Set
-        emit "\n========================\nLoading Edges\n========================\n  From DC?  Referenced?  Edge \n  %s  %s  %s" % ['--------', '-----------', '-'*45]
-        edges.each do |e|
-          # don't want to emit edge which is from a Data Connection to a 
-          # Calculated Field which is also referenced by another calculated field
-          isFromDC   = e.from.type == :TwbDataConnection 
-          isRefField = @referencedFields.include?(e.to.id)
-          edgesAsStrings.add(e.dot) unless isFromDC && isRefField 
-        end
-        emit "------------------------\n "
-        edgesAsStrings.each do |es|
-          dotFile.puts "        #{es}"
-          emit "           #{es}"
-        end
-        emit "========================\n "
-        dotFile.puts ""
-        dotFile.puts "   // }"
-        dotFile.puts "\n\n  //  4--------------------------------------------------------------------"
-        # "table::JIRA_HARVEST_Correspondence__c::Jira" [label="JIRA_HARVEST_Correspondence__c"]
-        nodes = SortedSet.new
-        edges.each do |e|
-          nodes.add e.from.dotLabel
-          nodes.add e.to.dotLabel
-        end
-        nodes.each do |n|
-          dotFile.puts n
-        end
-        dotFile.puts "\n\n  //  5--------------------------------------------------------------------"
-        emitTypes(      edges,   dotFile )
-        rankRootFields( dotFile, rootFields )
-        closeDot(       dotFile, twb )
-        # renderPng(twb.name,dotFileName)
-        # renderPdf(twb.name,dotFileName)
-        renderDot(twb,dotFileName,'pdf')
-        renderDot(twb,dotFileName,'png')
-        renderDot(twb,dotFileName,'svg')
-        emitEdges edges
-      end
-
-
-      def graphEdges twb, edges
-        graphFile = File.new(twb + '.cypher', 'w')
-        # graphFile.puts "OKEY DOKE, graphing away"
-        cypherCode = Set.new
-        edges.each do |edge|
-          cypherCode.add edge.from.cypherCreate
-          cypherCode.add edge.to.cypherCreate
-          cypherCode.add edge.cypherCreate
-        end
-        cypherCode.each do |cc|
-          graphFile.puts cc
-        end
-        graphFile.puts "\nreturn *"
-        graphFile.close unless graphFile.nil?
-        @imageFiles << File.basename(graphFile)
-      end
-
-      def emitEdges edges
-        emit  "  %-15s    %s" % ['type', 'Edge']
-        emit  "  %-15s    %s" % ['-'*15, '-'*35]
-        edges.each do |edge|
-          emit  "  %-15s    %s" % [edge.from.type, edge.from]
-          emit  "  %-15s    %s" % [edge.to.type,   edge.to]
-          emit  "\n "
-        end
-      end
-
-      def emitTypes edges, dotFile
-        typedNodes = {}
-        dotFile.puts "\n\n  //  2--------------------------------------------------------------------"
-        edges.each do |edge|
-               emit  "   EDGE  :: #{edge}"
-          loadNodeType typedNodes, edge.from
-          loadNodeType typedNodes, edge.to
-        end
-        typedNodes.each do |type, nodes|
-               emit  "+++++++++ typedNodes of '#{type}''  "
-          nodes.each do |node|
-                 emit  "           -n- #{node}"
-          end
-          rankSame(dotFile, type, nodes) unless type == :CalculatedField
-        end
-        # labelTypes dotFile, edges
-      end
-
-      def loadNodeType set, node
-        type = node.type
-        set[type] = Set.new unless set.include? type
-        set[type].add node
-      end
-
-      def rankSame dotFile, type, nodes
-        dotFile.puts "\n  // '#{type}' --------------------------------------------------------------------"
-        dotFile.puts "\n  {rank=same "
-        # dotFile.puts "     \"#{type}\"  [shape=\"box3d\" style=\"filled\" ]" unless ''.eql? type # [shape=\"box3d\"  style=\"filled\" ]\"" unless label.equal? ''
-        nodes.each do |node|
-          dotFile.puts "     \"#{node.id}\""
-        end
-        dotFile.puts "  }"
-      end
-
-      def rankRootFields dotFile, dsRootFields
-        dotFile.puts "\n  // Unreferenced (root) Calculated Fields -----------------------------------------"
-        dotFile.puts "\n  {rank=same "
-        dsRootFields.each do |rf|
-          dotFile.puts "     \"#{rf}\""
-        end
-        dotFile.puts "  }"
-      end
-
-
-      def labelTypes dotFile, edges
-        fromTos = Set.new
-        edges.each do |edge|
-          # fromTos.add "\"Alien Data Source\"  -> \"Alien Data Source\""
-          fromTos.add "\"#{edge.from.type}\""
-          fromTos.add "\"#{edge.to.type}\""
-        end
-        return if fromTos.empty?
-        dotFile.puts "\n  //  3--------------------------------------------------------------------"
-        dotFile.puts '   subgraph cluster_0 {'
-        dotFile.puts '     color=white;'
-        dotFile.puts '     node [shape="box3d"  style="filled" ];'
-        fromTos.each do |ft|
-          dotFile.puts "    #{ft}"
-        end
-        dotFile.puts '   }'
-      end
-
-
-      def emit(local=@localEmit, stuff)
-          #puts "\nstuff.class #{stuff.class}  :: #{stuff}" if local
-          if stuff.is_a? String then
-            lines = stuff.split(/\n/)
-            lines.each do |line|
-              @logger.debug "#{@emitPrefix}#{line}"
-                       puts "#{@emitPrefix}#{line}" if local
-            end
-          else
-              @logger.debug "#{@emitPrefix}#{stuff}"
-                       puts "#{@emitPrefix}#{stuff}" if local
-          end
-      end
-
-
-      def initDot twb
-        dotFile = File.open("#{twb}#{@@processName}.dot",'w')
-        dotFile.puts @@dotHeader
-        return dotFile
-      end
-
-      def closeDot dotFile, twb
-        dotFile.puts ' '
-        dotFile.puts '// -------------------------------------------------------------'
-        dotFile.puts ' '
-        dotFile.puts '   subgraph cluster_1 {'
-        # dotFile.puts '     color=white;'
-        dotFile.puts '     style=invis;'
-        # dotFile.puts '     border=0;'
-        dotFile.puts '     node [border=blue];'
-        dotFile.puts ' '
-        dotFile.puts '     "" [style=invis]'
-        dotFile.puts "     \"Tableau Tools\\nCalculated Fields Map\\nWorkbook '#{twb}'\\n#{Time.new.ctime}\"  [penwidth=0]"
-        # dotFile.puts "     \"Tableau Tools Workbook  Calculated Fields Map\\n#{Time.new.ctime}\"  -> \"\"  [style=invis]"
-        dotFile.puts ' '
-        dotFile.puts '   }'
-        dotFile.puts ' '
-        dotFile.puts '}'
-        dotFile.close
-      end
-
-
-      def renderDot twb, dot, format
-             emit  "Rendering DOT file\n  - #{twb}\n  - #{dot}\n  - #{format}"
-        imageType = '-T' + format
-        imageFile  = twb + @@processName + 'Graph.' + format
-        imageParam = '-o' + imageFile
-             emit  "system #{@@gvDotLocation} #{imageType} #{imageParam} #{dot}"
-                     system  @@gvDotLocation,   imageType,   imageParam,   dot
-        @imageFiles << imageFile
-        return imageFile
-      end
-
-
-  end #  class
-
-end   # module Analysis
-end   # module Twb