twb 4.9.10 → 5.2.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: ae42176483937996d559e926704bfeb458797941bf1b886ab07a0826589c6a47
-  data.tar.gz: cb1995d3c14a0db492631bb00f2d635b0a2abd2df3b04e7dee4639a95e7fe84d
+  metadata.gz: ebe8daee637317b5fb8024b346e2eed50883e00839a0d21a031672a55e116f44
+  data.tar.gz: 509985403a45d56d8b3fc3690d3d14c934700fe6879d356de9579cd3df6792bc
 SHA512:
-  metadata.gz: ee6e513b77ddbfc5eaa758ddc1281c78cad2ce20eb376d7144975d1e05371bef9e05ba32df08d0f587777113bcfa3242e0508c891e97d0690d8a31602740deed
-  data.tar.gz: dc69868cb98f02ca25ce5cf208ca935c3e16c4a6b18e36dbc90cada288f864aeb7df90df9d891545177a0c66c3f0edf24c2cac711d37797b47c1fd98b4693b74
+  metadata.gz: da4d761cceb4011cdd9442ef2ce959a3d74e9b2d1cdc004ef484c6418bfa7bc6d48e1848950a11d73d3af403de7ded1c8a3b4b07b8262b881ac9f2cab0c3c05e
+  data.tar.gz: 50fab8909339cb928ed2d717409b66a965c1588b96c47100842e8f667c7a85bdf9c680f66482ca6244ccd23437998b623c4885f2970a8e5a00539a2254560d1d

data/lib/twb.rb

@@ -54,6 +54,7 @@ require_relative 'twb/analysis/documentedfieldsmarkdownemitter'
 require_relative 'twb/analysis/annotatedfieldscsvemitter'
 require_relative 'twb/analysis/workbooksummaryanalyzer'
 require_relative 'twb/analysis/calculatedfields/calculatedfieldsanalyzer'
+require_relative 'twb/analysis/calculatedfields/dotanalyzer'
 require_relative 'twb/analysis/calculatedfields/groupfieldsanalyzer'
 require_relative 'twb/analysis/calculatedfields/markdownemitter'
 require_relative 'twb/analysis/calculatedfields/csvemitter'
@@ -80,5 +81,5 @@ require_relative 'twb/analysis/sheets/sheetsintooltipanalyzer'
 # Represents Tableau Workbooks, their contents, and classes that analyze and manipulate them.
 #
 module Twb
-  VERSION = '4.9.10'
+  VERSION = '5.2.3'
 end

data/lib/twb/analysis/calculatedfields/calculatedfieldsanalyzer.rb

@@ -78,12 +78,12 @@ module Analysis
       @fieldTables = {}
 
 
-      @@dotHeader = <<DOTHEADER
-  digraph g {
-      graph [rankdir="LR" splines=line];
-      node  [shape="box"  width="2"];
+#       @@dotHeader = <<DOTHEADER
+#   digraph g {
+#       graph [rankdir="LR" splines=line];
+#       node  [shape="box"  width="2"];
 
-DOTHEADER
+# DOTHEADER
 
       def initialize(**args)
           emit "initialize CalculatedFieldsAnalyzer args #{args}"
@@ -141,7 +141,7 @@ DOTHEADER
           # end
           processDataSource ds
         end
-        mapTwb
+        # mapTwb
         emitGml
         @twbCount += 1
         finis
@@ -195,10 +195,10 @@ DOTHEADER
               calculatedFields.add calcField.id
               dsFields[calcField.uiname] = calcField
               # if @doGraph
-                calcFieldNode    = Twb::Util::Graphnode.new(name: calcField.uiname, id: calcField.id, type: calcField, properties: {:DataSource => ds.uiname})
-                @nodes.add calcFieldNode
-                dsFieldEdge     = Twb::Util::Graphedge.new(from: dataSourceNode, to: calcFieldNode, relationship: 'contains')
-                @edges.add dsFieldEdge
+              calcFieldNode    = Twb::Util::Graphnode.new(name: calcField.uiname, id: calcField.id, type: calcField, properties: {:DataSource => ds.uiname})
+              @nodes.add calcFieldNode
+              dsFieldEdge     = Twb::Util::Graphedge.new(from: dataSourceNode, to: calcFieldNode, relationship: 'contains')
+              @edges.add dsFieldEdge
               # end
               calculation     = calcField.calculation
               if calculation.has_formula
@@ -249,47 +249,26 @@ DOTHEADER
                                          ]
                   end
                   #-- collect fields referenced in formula
-                  emit "# Calculated Fields: #{calculation.calcFields.length}"
-                  calculation.calcFields.each do |rf|
-                    emit "  referenced field       ::'#{rf}'"
-                    emit "  referenced field.name  ::'#{rf.name.nil?}'  :: '#{rf.name}'"
-                    emit "  referenced field.uiname::'#{rf.uiname}'"
-                    # if @doGraph
-                    unless rf.uiname.nil?
-                      properties = {'DataSource' => ds.uiname, 'DataSourceReference' => 'local', :source => rf}
-                      refFieldNode  = Twb::Util::Graphnode.new(name: rf.uiname, id: rf.id,   type: rf.type, properties: properties)
-                      @nodes.add refFieldNode
-                      fieldFieldEdge = Twb::Util::Graphedge.new(from: calcFieldNode,  to: refFieldNode, relationship: 'references')
-                      @edges.add fieldFieldEdge
-                    end
-                    # end
-                    referencedFields.add rf.id
-                    refFieldTable = ds.fieldTable(rf.name)
-                    emit "refFieldTable.nil? : #{refFieldTable.nil?}"
-                    unless refFieldTable.nil?
-                      tableID        = refFieldTable + ':::' + ds.uiname
-                      tableName      = "||#{refFieldTable}||"
-                      # if @doGraph
-                        tableNode      = Twb::Util::Graphnode.new(name: tableName,      id: tableID,   type: :DBTable, properties: properties)
-                        @nodes.add tableNode
-                        fieldFieldEdge = Twb::Util::Graphedge.new(from: refFieldNode,  to: tableNode, relationship: 'is a field in')
-                        @edges.add fieldFieldEdge
-                      # end
-                      # fldToDsNode    = tableNode
-                    end
+                  emit "# Calculated Fields: #{calculation.referencedFields.length}"
+                  calculation.referencedFields.each do |rf|
+                    emit "  referenced field       :: %12s  %s " % [ rf.dataSourceName, rf.uiname ]
                     @csvFormulaFields <<  [ 
                                             @referencedFieldsCount += 1,
                                             @twb.name,
                                             # @modTime,
-                                            ds.uiname,
+                                            rf.dataSourceName, # ds.uiname,
                                             calcField.uiname,
                                             calculation.formulaFlat,
                                             calculation.formulaFlatResolved,
                                             rf.name,
-                                            rf.uiname,
-                                            rf.id,
-                                            refFieldTable
+                                            rf.uiname,   #.uiname,
+                                            '',  # rf.id,
+                                            '',  #refFieldTable
                                           ]
+                    refFieldNode    = Twb::Util::Graphnode.new(name: rf.uiname, id: rf.id, type: rf, properties: {:DataSource => ds.uiname})
+                    @nodes.add refFieldNode
+                    refFieldEdge     = Twb::Util::Graphedge.new(from: calcFieldNode, to: refFieldNode , relationship: 'references')
+                    @edges.add refFieldEdge
                   end # resolvedFields.each do
               end # if calculation.has_formula
             end # ds.calculatedFields.each 
@@ -330,49 +309,49 @@ DOTHEADER
             emit "\t       formula:: #{calculation.formulaFlat}"
           end
 
-          def mapTwb 
-            twb        = @twb.name
-            rootFields = @twbRootFields
-            dotStuff    = initDot twb
-            dotFile     = dotStuff[:file]
-            dotFileName = dotStuff[:name]
-            dotFile.puts "\n //  subgraph cluster_1 {"
-            dotFile.puts "   //    color= grey;"
-            dotFile.puts ""
-            edgesAsStrings = SortedSet.new
-            # this two step process coalesces the edges into a unique set, avoiding duplicating the dot
-            # file entries, and can be shrunk when graph edges expose the bits necessary for management by Set
-            emit "\n========================\nLoading Edges\n========================\n  From DC?  Referenced?  Edge \n  %s  %s  %s" % ['--------', '-----------', '-'*45]
-            @edges.each do |e|
-              # don't want to emit edge which is from a Data Connection to a
-              # Calculated Field which is also referenced by another calculated field
-              isFromDC   = e.from.type == :TwbDataConnection
-              isRefField = @referencedFields.include?(e.to.id)
-              edgesAsStrings.add(e.dot) unless isFromDC && isRefField
-              # emit "   ES        #{e.dot}"
-              # emit "   ES  from  #{e.from}"
-              # emit "   ES    to  #{e.to}"
-            end
-            emit "------------------------\n "
-            edgesAsStrings.each do |es|
-              dotFile.puts "        #{es}"
-            end
-            emit "========================\n "
-            dotFile.puts ""
-            dotFile.puts "   // }"
-            dotFile.puts "\n\n  //  4 NODES --------------------------------------------------------------------"
-            @nodes.each do |n|
-              dotFile.puts n.dotLabel
-            end
-            dotFile.puts "\n\n  //  5--------------------------------------------------------------------"
-            emitTypes(      dotFile )
-            closeDot(       dotFile, twb )
-            emit "Rendering DOT file - #{twb}"
-            renderDot(twb,dotFileName,'pdf')
-            renderDot(twb,dotFileName,'png')
-            renderDot(twb,dotFileName,'svg')
-            # emitEdges
-          end
+          # def mapTwb 
+          #   twb        = @twb.name
+          #   rootFields = @twbRootFields
+          #   dotStuff    = initDot twb
+          #   dotFile     = dotStuff[:file]
+          #   dotFileName = dotStuff[:name]
+          #   dotFile.puts "\n //  subgraph cluster_1 {"
+          #   dotFile.puts "   //    color= grey;"
+          #   dotFile.puts ""
+          #   edgesAsStrings = SortedSet.new
+          #   # this two step process coalesces the edges into a unique set, avoiding duplicating the dot
+          #   # file entries, and can be shrunk when graph edges expose the bits necessary for management by Set
+          #   emit "\n========================\nLoading Edges\n========================\n  From DC?  Referenced?  Edge \n  %s  %s  %s" % ['--------', '-----------', '-'*45]
+          #   @edges.each do |e|
+          #     # don't want to emit edge which is from a Data Connection to a
+          #     # Calculated Field which is also referenced by another calculated field
+          #     isFromDC   = e.from.type == :TwbDataConnection
+          #     isRefField = @referencedFields.include?(e.to.id)
+          #     edgesAsStrings.add(e.dot) unless isFromDC && isRefField
+          #     # emit "   ES        #{e.dot}"
+          #     # emit "   ES  from  #{e.from}"
+          #     # emit "   ES    to  #{e.to}"
+          #   end
+          #   emit "------------------------\n "
+          #   edgesAsStrings.each do |es|
+          #     dotFile.puts "        #{es}"
+          #   end
+          #   emit "========================\n "
+          #   dotFile.puts ""
+          #   dotFile.puts "   // }"
+          #   dotFile.puts "\n\n  //  4 NODES --------------------------------------------------------------------"
+          #   @nodes.each do |n|
+          #     dotFile.puts n.dotLabel
+          #   end
+          #   dotFile.puts "\n\n  //  5--------------------------------------------------------------------"
+          #   emitTypes(      dotFile )
+          #   closeDot(       dotFile, twb )
+          #   emit "Rendering DOT file - #{twb}"
+          #   renderDot(twb,dotFileName,'pdf')
+          #   renderDot(twb,dotFileName,'png')
+          #   renderDot(twb,dotFileName,'svg')
+          #   # emitEdges
+          # end
 
           def cypher twbName
             if @doGraph
@@ -488,44 +467,44 @@ DOTHEADER
             dotFile.puts '   }'
           end
 
-          def initDot twb
-            dotFileName = docFile("#{twb}#{@@processName}.dot")
-            dotFile = File.open(dotFileName,'w')
-            dotFile.puts @@dotHeader
-            return {:file => dotFile, :name => dotFileName}
-          end
+          # def initDot twb
+          #   dotFileName = docFile("#{twb}#{@@processName}.dot")
+          #   dotFile = File.open(dotFileName,'w')
+          #   dotFile.puts @@dotHeader
+          #   return {:file => dotFile, :name => dotFileName}
+          # end
 
-          def closeDot dotFile, twb
-            dotFile.puts ' '
-            dotFile.puts '// -------------------------------------------------------------'
-            dotFile.puts ' '
-            dotFile.puts '   subgraph cluster_1 {'
-            # dotFile.puts '     color=white;'
-            dotFile.puts '     style=invis;'
-            # dotFile.puts '     border=0;'
-            dotFile.puts '     node [border=blue];'
-            dotFile.puts ' '
-            dotFile.puts '     "" [style=invis]'
-            dotFile.puts "     \"Tableau Tools\\nCalculated Fields Map\\nWorkbook '#{twb}'\\n#{Time.new.ctime}\"  [penwidth=0]"
-            # dotFile.puts "     \"Tableau Tools Workbook  Calculated Fields Map\\n#{Time.new.ctime}\"  -> \"\"  [style=invis]"
-            dotFile.puts ' '
-            dotFile.puts '   }'
-            dotFile.puts ' '
-            dotFile.puts '}'
-            dotFile.close
-          end
+          # def closeDot dotFile, twb
+          #   dotFile.puts ' '
+          #   dotFile.puts '// -------------------------------------------------------------'
+          #   dotFile.puts ' '
+          #   dotFile.puts '   subgraph cluster_1 {'
+          #   # dotFile.puts '     color=white;'
+          #   dotFile.puts '     style=invis;'
+          #   # dotFile.puts '     border=0;'
+          #   dotFile.puts '     node [border=blue];'
+          #   dotFile.puts ' '
+          #   dotFile.puts '     "" [style=invis]'
+          #   dotFile.puts "     \"Tableau Tools\\nCalculated Fields Map\\nWorkbook '#{twb}'\\n#{Time.new.ctime}\"  [penwidth=0]"
+          #   # dotFile.puts "     \"Tableau Tools Workbook  Calculated Fields Map\\n#{Time.new.ctime}\"  -> \"\"  [style=invis]"
+          #   dotFile.puts ' '
+          #   dotFile.puts '   }'
+          #   dotFile.puts ' '
+          #   dotFile.puts '}'
+          #   dotFile.close
+          # end
 
 
-          def renderDot twb, dot, format
-            imageType = '-T' + format
-            imageFile  = './ttdoc/' + twb + @@processName + 'Graph.' + format
-            imageParam = '-o"' + imageFile + '"'
-            emit  "system  #{@@gvDotLocation} #{imageType} #{imageParam} \"#{dot}\""
-                   system "#{@@gvDotLocation} #{imageType} #{imageParam} \"#{dot}\""
-            emit  "           - #{imageFile}"
-            @imageFiles << imageFile
-            return imageFile
-          end
+          # def renderDot twb, dot, format
+          #   imageType = '-T' + format
+          #   imageFile  = './ttdoc/' + twb + @@processName + 'Graph.' + format
+          #   imageParam = '-o"' + imageFile + '"'
+          #   emit  "system  #{@@gvDotLocation} #{imageType} #{imageParam} \"#{dot}\""
+          #          system "#{@@gvDotLocation} #{imageType} #{imageParam} \"#{dot}\""
+          #   emit  "           - #{imageFile}"
+          #   @imageFiles << imageFile
+          #   return imageFile
+          # end
 
   end #  class
 

data/lib/twb/analysis/calculatedfields/dotanalyzer.rb

@@ -0,0 +1,139 @@
+#  dotanalyzer.rb - this Ruby script Copyright 2017, 2018 Christopher Gerrard
+#
+#  This program is free software: you can redistribute it and/or modify
+#  it under the terms of the GNU General Public License as published by
+#  the Free Software Foundation, either version 3 of the License, or
+#  (at your option) any later version.
+#
+#  This program is distributed in the hope that it will be useful,
+#  but WITHOUT ANY WARRANTY; without even the implied warranty of
+#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#  GNU General Public License for more details.
+#
+#  You should have received a copy of the GNU General Public License
+#  along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+
+module Twb
+module Analysis
+module CalculatedFields
+
+  class DotAnalyzer
+    include TabTool
+
+    attr_reader :docFileName
+
+    @@gvDotLocation = 'C:\\tech\\graphviz\\Graphviz2.38\\bin\\dot.exe'
+    @@imageTypes = ['pdf', 'png', 'svg']
+
+    def initialize(**args)
+      @args = args
+      init
+      @funcdoc = {:class=>self.class, :blurb=>'Create Dot files documenting Calculated Fields', :description=>'Analyze Calculated Fields - create Dot files' }
+      @metrics = {}
+      @imageFiles = Array.new
+    end
+
+    def processTWB twb
+      # twb = File.basename(twb)
+      @twb         = twb #Twb::Workbook.new twb
+      addDocFile @dotFile, @dotFileName, "Dot file of Calculated fields for Workbook '#{@twb.name}'"
+      @twb.datasources.each do |ds|
+        unless ds.calculatedFields.empty?
+          initDotFile ds.uiname
+          # @dotFile.puts "\n ## #{ds.uiname} \n "
+          # @dotFile.puts "__has #{ds.calculatedFields.length} calculated fields__\n "
+          @cfCnt = 0
+          calcFields = Set.new
+          refFields  = Set.new
+          edges      = Set.new
+          ds.calculatedFields.each do |cf|
+            @cfCnt += 1
+            calcFields << cf.uiname
+            edges            << " \"#{ds.uiname}\" -> \"#{cf.uiname}\"  [tailport=e, headport=w] "
+            cf.referencedFields.each do |rf|
+              refFields << rf.uiname
+              edges            << " \"#{cf.uiname}\" -> \"#{rf.uiname}\" [tailport=e, headport=w] "
+            end
+          end # ds.calculatedFields.each
+          # "federated.17h7owt0rsacke17cql8o0w2ittk"  ->  "New AO Actuals Query in PP+ (AO Variance Data)::vs Prior Year [YTD]"
+          # "federated.01s5lca037ted31gxs9sg0t9mnnt" [label="Controls"    ]
+          edges.each do |edge|
+            @dotFile.puts "\t #{edge.strip}"
+          end
+          @dotFile.puts " "
+          allFields = calcFields + refFields
+          allFields.each do |f|
+            @dotFile.puts "\t \"#{f}\" [label=\"#{f}\"]"
+          end
+          endPointFields = allFields - calcFields
+          rankSame(endPointFields) unless endPointFields.nil? || endPointFields.empty?
+          closeDotFile
+          @@imageTypes.each do |type|
+            renderDot type
+          end
+        end
+      end  # twb.datasources.each
+      finis
+    end    # def processTwb twb
+
+    private
+
+    def initDotFile dsName
+      @dotFileName = './ttdoc/' + @twb.name + '.' + dsName + '.CalculatedFields.dot'
+      @dotFile     = File.open(@dotFileName,'w')
+      # @dotFile.puts @@dotHeader
+      @dotFile.puts ' digraph g {'
+      @dotFile.puts '   graph [rankdir="LR" splines=line];'
+      @dotFile.puts '   node  [shape="box"  width="2"];'
+      @dotFile.puts ' '
+      @dotFile.puts '   subgraph cluster_0 {'
+    end
+
+    def closeDotFile
+      # @dotFile.puts "\n # counted #{@cfCnt} calculated fields\n "
+      # @dotFile.puts "\n }"
+     @dotFile.puts '   }'
+      @dotFile.puts ' '
+      @dotFile.puts '// -------------------------------------------------------------'
+     @dotFile.puts ' '
+     @dotFile.puts '   subgraph cluster_1 {'
+      #@dotFile.puts '     color=white;'
+     @dotFile.puts '     style=invis;'
+      #@dotFile.puts '     border=0;'
+     @dotFile.puts '     node [border=blue];'
+     @dotFile.puts ' '
+     @dotFile.puts '     "" [style=invis]'
+     @dotFile.puts "     \"Tableau Tools\\nCalculated Fields Map\\nWorkbook '#{@twb.name}'\\n#{Time.new.ctime}\"  [penwidth=0]"
+      #@dotFile.puts "     \"Tableau Tools Workbook  Calculated Fields Map\\n#{Time.new.ctime}\"  -> \"\"  [style=invis]"
+     @dotFile.puts ' '
+     @dotFile.puts '   }'
+     @dotFile.puts ' '
+     @dotFile.puts '}'
+      @dotFile.close
+    end
+
+      def rankSame fields
+        @dotFile.puts "\n {rank=same "
+        fields.each do |f|
+          @dotFile.puts "\t \"#{f}\" "
+        end
+        @dotFile.puts " } "
+      end
+
+      def renderDot format
+        imageType = '-T' + format
+        imageFile  = @dotFileName + '.Graph.' + format
+        imageParam = '-o"' + imageFile + '"'
+        emit "system  #{@@gvDotLocation} #{imageType} #{imageParam} \"#{@dotFileName}\""
+        system "#{@@gvDotLocation} #{imageType} #{imageParam} \"#{@dotFileName}\""
+        # emit  "           - #{imageFile}"
+        @imageFiles << imageFile
+        return imageFile
+      end
+
+  end # class DotAnalyzer
+
+end   # nodule CalculatedFields
+end   # module Analysis
+end   # module Twb

data/lib/twb/analysis/calculatedfields/markdownemitter.rb

@@ -52,10 +52,10 @@ module CalculatedFields
             @docFile.puts "#{l.gsub('<<','[').gsub('>>',']')}" 
           end
           @docFile.puts "```"
-          if cf.calcFields.length > 0
+          if cf.referencedFields.length > 0
             fieldsRefOrder  = []
             fieldsSortSet   = SortedSet.new
-            cf.calcFields.each do |field|
+            cf.referencedFields.each do |field|
               fieldsRefOrder.push field.uiname
               fieldsSortSet << field.uiname
             end

data/lib/twb/analysis/calculatedfields/t.rb

@@ -0,0 +1,6 @@
+require 'twb'
+
+twb = Twb::Workbook.new 'MS.twb'
+ds  = twb.datasources.to_a.last
+cfs =  ds.calculatedFields
+cfs.each { |cf|  puts "\n\n--\n#{cf.calculation.formula}\n--"; cf.calcFields.each { |f| puts "   %-12s %s " % [f[:ds], f[:field] ] } }

data/lib/twb/calculatedfield.rb

@@ -1,4 +1,4 @@
-#  Copyright (C) 2014, 2015, 2017  Chris Gerrard
+#  Copyright (C) 2014, 2015, 2020 Chris Gerrard
 #
 #  This program is free software: you can redistribute it and/or modify
 #  it under the terms of the GNU General Public License as published by
@@ -26,7 +26,7 @@ module Twb
     attr_reader :node,        :properties
     attr_reader :caption,     :name,       :uiname
     attr_reader :datatype,    :role,       :propType  
-    attr_reader :calculation, :calcFields 
+    attr_reader :calculation, :referencedFields 
     attr_reader :isGroup,     :groupMembers
     attr_reader :hidden
 
@@ -52,8 +52,8 @@ module Twb
       @properties ||= loadProperties
     end
 
-    def calcFields
-        @calculation.calcFields
+    def referencedFields
+        @calculation.referencedFields
     end
 
     def formulaLines

data/lib/twb/columnfield.rb

@@ -14,6 +14,7 @@
 #  along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
 require 'nokogiri'
+require 'pry'
 
 module Twb
 
@@ -85,7 +86,6 @@ module Twb
       @aggregation        = load 'aggregation'
       @autoColumn         = load 'auto-column'
       @datatypeCustomized = load 'datatype-customized'
-      # @calcField          = loadCalcField
     end
 
     def id
@@ -94,7 +94,7 @@ module Twb
 
     def load nodeName
       attr = @node.attribute(nodeName)
-      val  = attr.nil? ? nil : attr.text.strip.gsub(/^\[|\]$/,'')
+      val  = attr.nil? ? nil : attr.text.strip
       return val
     end
 

data/lib/twb/datasource.rb

@@ -56,7 +56,7 @@ module Twb
     attr_reader :tableFieldsMap
     attr_reader :fieldUINames
     attr_reader :aliases
-    attr_reader :calculatedFields, :calculatedFieldNamesMap, :calculatedFieldNames, :calculatedField 
+    attr_reader :calculatedFields, :calculatedFieldsMap, :calculatedFieldNames, :calculatedField 
     attr_reader :allFields 
     attr_reader :groups 
     attr_reader :filters
@@ -391,40 +391,6 @@ module Twb
       return @fieldUINames
     end
 
-    def loadFieldUINames
-      # puts 'loadFieldUINames'
-      @fieldUINames = {}
-      # puts 'metadataFields'
-      metadataFields.each do |fld|
-        # puts " - name:%-45s   |  uiname:%-45s   |  localName:%-45s " % [ "'#{fld.name}'", "'#{fld.uiname}'", "'#{fld.localName}'"]
-        @fieldUINames[fld.uiname]    = fld.uiname
-        @fieldUINames[fld.localName] = fld.uiname unless  fld.localName.nil?
-        @fieldUINames[fld.name]      = fld.uiname unless  fld.name.nil?
-      end
-      # puts 'calculatedFields'
-      calculatedFields.each do |fld| 
-        # puts " - name:%-45s   |  uiname:%-45s " % [ "'#{fld.name}'", "'#{fld.uiname}'"]
-        @fieldUINames[fld.name]   = fld.uiname
-        @fieldUINames[fld.uiname] = fld.uiname
-      end
-      # puts 'localFields'
-      localFields.each do |fld|
-        # puts " - name:%-45s   |  uiname:%-45s " % [ "'#{fld.name}'", "'#{fld.uiname}'"]
-        @fieldUINames[fld.name]   = fld.uiname
-        @fieldUINames[fld.uiname] = fld.uiname
-      end
-      # puts "columnFields: #{columnFields.length}"
-      columnFields.each do |fld|
-        @fieldUINames[fld.name]   = fld.uiname
-        @fieldUINames[fld.uiname] = fld.uiname
-      end
-      groups.each do |fld|
-        @fieldUINames[fld.name]   = fld.uiname
-        @fieldUINames[fld.uiname] = fld.uiname
-      end
-      return @fieldUINames
-    end
-
     def tableFields
       @tableFieldsMap.values
     end
@@ -482,8 +448,7 @@ module Twb
 
     def loadAllFields
       @allFields = SortedSet.new
-      dbf = dbFields
-      @allFields << dbf
+      @allFields << dbFields
       @allFields << calculatedFieldNames
     end
 
@@ -532,6 +497,53 @@ module Twb
 
     private
 
+    def loadFieldUINames
+      @fieldUINames = {}
+       # puts"metadataFields: #{metadataFields.length}"
+      metadataFields.each do |fld|
+         # puts" - name:%-45s   |  caption:%-45s   |  uiname:%-45s " % [ "'#{fld.name}'",  "'#{fld.caption}'", "'#{fld.uiname}'"]
+        unless fld.name.nil?
+          @fieldUINames[fld.uiname]    = fld.uiname
+          @fieldUINames[fld.localName] = fld.uiname unless  fld.localName.nil?
+          @fieldUINames[fld.name]      = fld.uiname
+        end
+      end
+       # puts"localFields: #{localFields.length}"
+      localFields.each do |fld|
+         # puts" - name:%-45s   |  caption:%-45s   |  uiname:%-45s " % [ "'#{fld.name}'",  "'#{fld.caption}'", "'#{fld.uiname}'"]
+        unless fld.caption.nil?
+          @fieldUINames[fld.name]   = fld.caption
+          @fieldUINames[fld.uiname] = fld.caption
+        end
+      end
+       # puts"groups: #{groups.length}"
+      groups.each do |fld|
+         # puts" - name:%-45s   |  caption:%-45s   |  uiname:%-45s " % [ "'#{fld.name}'",  "'#{fld.caption}'", "'#{fld.uiname}'"]
+        unless fld.caption.nil?
+          @fieldUINames[fld.name]   = fld.caption
+          @fieldUINames[fld.uiname] = fld.caption
+        end
+      end
+       # puts"calculatedFields: #{calculatedFields.length}"
+      calculatedFields.each do |fld| 
+         # puts" - name:%-45s   |  caption:%-45s   |  uiname:%-45s " % [ "'#{fld.name}'",  "'#{fld.caption}'", "'#{fld.uiname}'"]
+        unless fld.caption.nil?
+          @fieldUINames[fld.name]   = fld.caption
+          @fieldUINames[fld.uiname] = fld.caption
+        end
+      end
+       # puts"columnFields: #{columnFields.length}"
+      columnFields.each do |fld|
+         # puts" - name:%-45s   |  caption:%-45s   |  uiname:%-45s " % [ "'#{fld.name}'",  "'#{fld.caption}'", "'#{fld.uiname}'"]
+        unless fld.caption.nil?
+          # @fieldUINames[fld.name]                          = fld.caption
+          @fieldUINames[fld.name.gsub(/^[\[]|[\]]$/,'')]   = fld.caption
+          @fieldUINames[fld.uiname]                        = fld.caption
+        end
+      end
+      return @fieldUINames
+    end
+
     def loadGroups
       @groups = []
       groupNodes = @node.xpath('.//group')

data/lib/twb/fieldcalculation.rb

@@ -16,6 +16,7 @@
 require 'nokogiri'
 require 'digest/md5'
 require 'csv'
+require 'pry'
 
 module Twb
 
@@ -35,7 +36,7 @@ module Twb
     attr_reader :is_tableCalc
     attr_reader :is_lod,   :lodCodePos
     attr_reader :class,    :scopeIsolation
-    attr_reader :fields,   :remoteFields,  :calcFields
+    attr_reader :fields,   :remoteFields, :referencedFields
     attr_reader :comments, :uuid
 
     # attr_accessor :ttlogfile
@@ -120,59 +121,33 @@ module Twb
     end
 
     def formulaResolved
-      @formulaResolved ||= @formulaResolved = resolveFormula
+      @formulaResolved ||= resolveFormula
     end
     
     def resolveFormula
-      # puts "\ndef resolveFormula:\n--\n#{@formula}"
-      formula = @formula
-      parseFormFields # - extracts the fields from the formula; as persisted they're the internal names
-      @calcFields.each do |calcField|
-        if calcField.techUIdiff
-         # puts ":::: #{calcField.techCode}  //  #{calcField.uiCode}"
-          formula = formula.gsub(calcField.techCode,calcField.uiCode)
-         # puts ":--: #{formula}"
-        end
+      # emit "\ndef resolveFormula:\n--\n#{@formula}"
+      resolved = @formula
+      # emit "\t formula:#{resolved}:"
+      # parseFormFields # - extracts the fields from the formula; as persisted they're the internal names
+      referencedFields.each do |refField|
+        # emit "\t refField: "
+        resolved.gsub!(refField.techCode,refField.uiCode)
+        #   if calcField.techUIdiff
+        #    # puts ":::: #{calcField.techCode}  //  #{calcField.uiCode}"
+        #     formula = formula.gsub(calcField.techCode,calcField.uiCode)
+        #    # puts ":--: #{formula}"
+        #   end
       end
-      return formula
+      # emit "\t formula:#{resolved}:"
+      return resolved
     end
 
-    def calcFields
-      @calcFields ||= parseFormFields
+    def referencedFields
+      @referencedFields ||= parseFormFields
     end
 
-    def parseFormFields
-      # puts "--parseFormFields"
-      @fields     = Set.new    
-      @calcFields = Set.new
-      formula = @formulaFlat
-      if !formula.nil? && formula.include?('[') && formula.include?(']')
-          fields   = Set.new
-          # noSqLits =  formula.gsub( /'[\[\.\]]+'/, ' ')
-          quotes   =  formula.gsub('"',"'")
-          noSqLits =   quotes.gsub( /'[\[\.\]]+'/, ' ')
-          flatForm = noSqLits.gsub( /\n/, ' ')
-          stripFrt = flatForm.gsub( /^[^\[]*[\[]/    , '['      )
-          stripBck = stripFrt.gsub( /\][^\]]+$/      , ']'      )
-          stripMid = stripBck.gsub( /\][^\]]{2,}\[/  , ']]..[[' )
-          stripCom = stripMid.gsub( /\][ ]*,[ ]*\[/  , ']]..[[' )
-          stripFns = stripMid.gsub( /\][ ]*[\*\/+\-><,=][ ]*\[/  , ']]..[[' )
-          fields   = stripFns.split(']..[')
-          emit "::self::: #{self} :: #{__LINE__} :: fields:'#{fields.inspect}'" 
-          fields.each do |field|
-            emit "::self::: #{self} :: #{__LINE__} :: field:'#{field}'" 
-            cf = CalculationField.new( field.gsub(/^\[|\]$/, ''), @dataSource )
-            @calcFields.add cf
-            @fields.add field.gsub(/^\[|\]$/, '')
-          end
-      end
-      return @calcFields
-    end
 
     def formulaResolvedLines
-      # puts "\ndef formulaResolvedLines\n--\n#{formulaResolved}"
-      # puts "--\n#{formulaResolved.split(/\n|\r\n/)}"
-      # puts "--\n#{formulaResolved.split(/\n|\r\n/)}"
       formulaResolved.split(/\n|\r\n/)
     end
 
@@ -212,11 +187,149 @@ module Twb
       return comments.strip
     end
 
-  end  # class FieldCalculation
+    private
 
 
+    def pullString chars
+      delim1 = chars.shift
+      delim2 = delim1+delim1
+      field  = delim1
+      done   = false
+      until done | chars.empty?
+        s01 = chars[0..1].join
+        if !delim1.eql? chars[0]
+          field += chars.shift
+        else
+          case s01
+          when delim2
+            field += delim2
+            chars.shift(2)
+          when delim1
+            field += delim1
+            chars.shift
+            done = true
+          else
+            field += delim1
+            chars.shift
+            done = true
+          end
+        end
+      end
+    end
+
+    def pullField chars
+      # chars = str.split ''
+      done  = false
+      ds    = ''
+      field = ''
+      until done
+        s01 = chars[0..1].join
+        s02 = chars[0..2].join
+        if ']'.eql? chars[0]
+          case s01
+          when ']]'
+            field += ']]'
+            chars.shift(2)
+          when ']'
+            field += chars.shift
+            done = true
+          else
+            if '].['.eql?(s02)
+              ds = field + ']'
+              chars.shift(2)
+              # fldstr   = chars.join
+              field    = pullField(chars)[:field]
+              done     = true
+            else
+              field += ']'
+              chars.shift
+              done = true
+            end
+          end
+        else
+          field += chars[0]
+          chars.shift
+        end
+      end
+      # puts "field: '#{field}' \t\t ds: #{ds}"
+      return {:field => field.sub(/\[/,'').sub(/\]$/,''), :ds => ds.sub(/\[/,'').sub(/\]$/,'') }
+    end
 
-  class CalculationField
+    def parseFormFields # formula
+      @referencedFields = Array.new
+      rawFields         = Array.new
+      if !@formula.nil? && @formula.include?('[') && @formula.include?(']')
+      chars  = formula.split('')
+      until chars.empty?
+        char0 = chars[0]
+        case char0
+        when '"', "'"
+          pullString(chars)
+        when '['
+          rawFields << pullField(chars)
+        else
+          unless chars.nil? | chars.empty?
+            chars.shift
+          end
+        end
+      end
+      rawFields.each do |rf|
+        ds = rf[:ds]
+        dataSource = if ''.eql? ds
+                       @dataSource
+                     else 
+                       @dataSource.workbook.datasource(ds)
+                     end
+        # fieldUIName = dataSource.fieldUIName(rf[:field])
+        refField    = ReferencedField.new(rf[:field], dataSource)
+        @referencedFields << refField
+        end
+      end
+      return @referencedFields
+    end
+
+    def parseFormFieldsx # formula
+      rawFields = Set.new
+      if !@formula.nil? && @formula.include?('[') && @formula.include?(']')
+        noComms   = @formula.gsub(/\/\/.*\r\n/,' ')
+        formBase  = noComms.gsub(/\r\n/,' ')
+        formLen   = formBase.length
+        formChars = formBase.split '' 
+        until formChars.empty?
+          c = formChars.shift
+          case c
+            when '"', "'"  
+              pullString(formChars, c)
+            when '['       
+              rawFields << pullField(formChars, ']', @referencedFields)
+          end
+        end
+      end
+      @referencedFields = Set.new
+      rawFields.each do |rf|
+        # @referencedFields << rf
+        dataSource = if ''.eql? rf[:ds]
+                       @dataSource
+                     else 
+                       @dataSource.workbook.datasource(rf[:ds])
+                     end
+        # if dataSource.nil?
+        #   binding.pry
+        # end
+        fieldUIName = dataSource.fieldUIName(rf[:field])
+        # binding.pry
+        refField = ReferencedField.new(rf[:field], dataSource)
+        # binding.pry
+        @referencedFields << refField
+      end
+      return @referencedFields
+    end
+
+  end  # class FieldCalculation
+
+
+  # class CalculationField
+  class ReferencedField
     # is a field used in a calculation, resolved into its human-meaningful form
 
     include Comparable
@@ -228,65 +341,29 @@ module Twb
     attr_reader :fqName,     :type
     attr_reader :techUIdiff
 
-    def initialize code, datasource
-      # puts "\n\nCalculationField ::  %-25s  |  %s  " % [datasource.uiname, code]
+    def initialize name, datasource
+      # puts "\n\nReferencedField ::  ds: %-25s  |  n: %s  " % [datasource, name]
+      @name             = name
       @dataSource       = datasource
-      @dataSourceName   = datasource.uiname
+      @dataSourceName   = datasource.nil? ? nil : datasource.uiname
       @dataSourceRef    = :local
       @dataSourceExists = true
+      @techCode         = "[#{name}]"
       @techUIdiff       = false
-      @uiname           = ''
-      rawCode  = code.gsub(/^\[|\]$/,'')
-      parts    = rawCode.split('].[')
-      #puts "Field: #{code}  \t parts: #{parts.length} - #{parts.inspect}"
-      if parts.length == 1
-        @name = parts[0]
-        @techCode = "[#{parts[0]}]"
-        #puts "@name: #{@name}"
-        if datasource.nil?
-          # puts 'a'
-          @uiname = @name
-          @uiCode = @techCode
-          @techUIdiff = false
-        else # !datasource.nil?
-          # puts 'b'
-          #puts "b - found uiname for '#{@name}'?: #{!datasource.fieldUIName(@name).nil?} \t is:#{datasource.fieldUIName(@name)} "
-          @uiname = datasource.fieldUIName(@name).nil? ? @name : datasource.fieldUIName(@name) 
-          @uiCode = @uiname.nil? ? @techCode :  "[#{@uiname}]"
-          @techUIdiff = !@techCode.eql?(@uiCode)
-          # puts ":b #{datasource.fieldUIName(@name).nil?}  ... #{@name}  ... #{@uiname}"
-          # puts "CalculationField ::       uin: %-25s | @name:%-s" % [@uiname,@name]
-        end
-      else # parts.length <> 1
-        # puts 'c'
-        rdstech   = parts[0]
-        calcField = parts[1]
-        @uiname   = calcField
-        @dataSourceName = rdstech
-        @dataSourceRef  = :remote
-        @techCode = "[#{rdstech}].[#{calcField}]"
-        workbook  = datasource.workbook
-        @dataSource  = workbook.nil? ? nil : workbook.datasource(rdstech)
-       # puts "\t  twb: #{workbook.class}  /  remoteds: #{remoteds.class}  : #{remoteds.nil? ? "<<NOT FOUND:#{rdstech}:>>" : remoteds.uiname} "
-        #--
-        if @dataSource.nil? || @dataSource.fieldUIName(calcField).nil?
-          # puts 'd'
-          @uiname     = calcField
-          @uiCode     = "[<<NOT FOUND>>#{rdstech}].[#{calcField}]"
-          @techUIdiff = true
-          @dataSourceExists = false
-        else  # !remoteds.nil?
-          # puts 'e'
-          @dataSourceName = @dataSource.uiname
-          @uiname         = @dataSource.fieldUIName(calcField)
-          @uiCode         = "[#{@dataSourceName}].[#{@uiname}]"
-          @techUIdiff     = !@techCode.eql?(@uiCode)
-          @dataSourceExists = true
-        end
+      if dataSource.nil?
+        # puts 'a'
+        @uiname = @name
+        @uiCode = @techCode
+        @techUIdiff = false
+      else # !datasource.nil?
+        # puts 'b'
+        # puts "b - found uiname for '#{@name}'?: #{!datasource.fieldUIName(@name).nil?} \t is:#{datasource.fieldUIName(@name)} "
+        @uiname = datasource.fieldUIName(@name).nil? ? @name : datasource.fieldUIName(@name) 
+        @uiCode = @uiname.nil? ? @techCode :  "[#{@uiname}]"
+        @techUIdiff = !@techCode.eql?(@uiCode)
+        # puts ":b #{datasource.fieldUIName(@name).nil?}  ... #{@name}  ... #{@uiname}"
+        # puts "CalculationField ::       uin: %-25s | @name:%-s" % [@uiname,@name]
       end
-      # puts "\t     dsName: #{@dataSourceName}"
-      # puts "\t   @name: #{@name}"
-      # puts "\t     uiname: #{@uiname}"
       @fqName = "#{@dataSourceName}::#{@uiname}"
       @type   = if @dataSource.nil?
                    :CalculatedField
@@ -308,6 +385,6 @@ module Twb
       @fqName <=> other.fqName
     end
 
-  end # class CalculationField
+  end # class ReferencedField
 
 end  # module Twb

data/lib/twb/quickfilter.rb

@@ -120,7 +120,7 @@ module Twb
     # filter class='relative-date' column='[Sample - Superstore].[none:Order Date:qk]' first-period='1' include-future='true' include-null='false' last-period='1' period-type='year' />
     def resolveRelativeDate
       emit "resolveRelativeDate"
-      periodType  = @node['period-type']
+      periodType  = @node['period-type'].nil? ? '' : @node['period-type']
       inclFuture  = @node['include-future'] == 'true'
       firstPeriod = @node['first-period'].to_i
       lastPeriod  = @node['last-period'].to_i

data/test/testFieldCalculation.rb

@@ -16,7 +16,9 @@
 require 'nokogiri'
 
 #require 'twb'
-require 'C:\tech\Tableau\tools\Ruby\gems\twb\lib\twb.rb'
+# require 'C:\tech\Tableau\tools\Ruby\gems\twb\lib\twb.rb'
+# require 'C:\tech\Tableau\Tableau Tools\Ruby\gems\twb\lib\twb.rb'
+require 'twb'
 require "test/unit"
 
 system "cls"
@@ -25,10 +27,11 @@ class TestFieldCalculation < Test::Unit::TestCase
 
   def test_fragment1
     doc = Nokogiri::XML::Document.parse <<-EOHTML
-<calculation class='tableau' formula='abc' />
+<calculation class='tableau' name='I am a formular field' formula='abc' datatype='datatype' role='' type='' class='calcfield' />
 EOHTML
-    calcNode = doc.at_xpath('./calculation')
-    calc = Twb::FieldCalculation.new(calcNode)
+    calcNode  = doc.at_xpath('./calculation')
+    calcField = Twb::CalculatedField.new calcNode
+    calc = Twb::FieldCalculation.new(calcField)
     assert(!calc.nil?)
     #puts "node: #{calcNode}"
     #puts "formula: #{calc.formula}"
@@ -41,10 +44,11 @@ EOHTML
 
   def test_fragment2
     doc = Nokogiri::XML::Document.parse <<-EOHTML
-        <calculation class='tableau' formula='// this is the number of days between the order and shipment&#13;&#10;&#13;&#10;datediff(&apos;day&apos;,[Order Date] , [other].[Ship Date])' />
+        <calculation class='tableau' name='Another formula fied' formula='// this is the number of days between the order and shipment&#13;&#10;&#13;&#10;datediff(&apos;day&apos;,[Order Date] , [other].[Ship Date])' />
 EOHTML
     calcNode = doc.at_xpath('./calculation')
-    calc = Twb::FieldCalculation.new(calcNode)
+    calcField = Twb::CalculatedField calcNode
+    calc = Twb::FieldCalculation.new(calcField)
     assert(!calc.nil?)
     #puts "node: #{calcNode}"
     #puts "formula: #{calc.formula}"

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: twb
 version: !ruby/object:Gem::Version
-  version: 4.9.10
+  version: 5.2.3
 platform: ruby
 authors:
 - Chris Gerrard
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2020-05-04 00:00:00.000000000 Z
+date: 2020-08-06 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: creek
@@ -64,9 +64,11 @@ files:
 - lib/twb/analysis/annotatedfieldscsvemitter.rb
 - lib/twb/analysis/calculatedfields/calculatedfieldsanalyzer.rb
 - lib/twb/analysis/calculatedfields/csvemitter.rb
+- lib/twb/analysis/calculatedfields/dotanalyzer.rb
 - lib/twb/analysis/calculatedfields/fieldsaliasesanalyzer.rb
 - lib/twb/analysis/calculatedfields/groupfieldsanalyzer.rb
 - lib/twb/analysis/calculatedfields/markdownemitter.rb
+- lib/twb/analysis/calculatedfields/t.rb
 - lib/twb/analysis/datasources/categoricalcolorcodinganalyzer.rb
 - lib/twb/analysis/datasources/datasourcefieldsanalyzer.rb
 - lib/twb/analysis/datasources/datasourcefieldscsvemitter.rb