twb 4.7.1 → 4.9.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 9ba8a4bcf6b220aaf9d3b94a91a903ec055a1637d7d78c063a58a5def3027844
-  data.tar.gz: a7f76a9e0fb057cf7aee2c38c82b974e0a59575eb955bd41636e79a38a251f65
+  metadata.gz: 39d006e0c059121c7c53795a7b2c7b90c425a3809bb0287b215f5950ac4187b8
+  data.tar.gz: a49a8f7fb9cbf9a206def1cdc1faf649455dea27d180c88d64e518ae1da3b4e2
 SHA512:
-  metadata.gz: 685229ff4ae9181176c42fb85cf9900e64823fa2cbcf4a8ef49a48b0ebbfa96ecf491e27e88c545478a2e4ccad0ba314f648baa28406964032d39e3daefe0904
-  data.tar.gz: 86eb832a11d3bb87a66835f29d5eb312477f55421e007dea39c8178568573d39f0ff5d529ab22dfbfe53128b6a1b1145948b5f92bd785cc780b57c3df76d7137
+  metadata.gz: 8f2983b81057ba13bfa029b4103fac22728f0fe34a7e41346373019161d7f6dc9857a54f67f071ae23f531e6a2a140448bc3c8e9ed95577b0c35ec4bd2e560d8
+  data.tar.gz: 9397a53a76c078a36dbfffa170c4a8f91b85a76d5c288e20c38e5a84ea9a3f78d9e7e01488f85d1c917dc6ce4f371671066367550723bd13ba79afa7f98b2b1c

data/lib/twb.rb

@@ -37,6 +37,7 @@ require_relative 'twb/quickfilter'
 # require_relative 'twb/docdashboardimagevert'
 # require_relative 'twb/docdashboardwebvert'
 require_relative 'twb/util/twbdashsheetdatadotbuilder'
+require_relative 'twb/util/csvrecordsatomizer'
 require_relative 'twb/util/dotfilerenderer'
 # require_relative 'twb/util/htmllistcollapsible'
 require_relative 'twb/util/xraydashboards'
@@ -59,11 +60,16 @@ require_relative 'twb/analysis/calculatedfields/csvemitter'
 # require_relative 'twb/analysis/datasources/datasourcefieldscsvemitter'
 require_relative 'twb/analysis/datasources/datasourcesenumerator.rb'
 require_relative 'twb/analysis/datasources/datasourcefieldsanalyzer'
-require_relative 'twb/analysis/datasources/datasourcetablefieldscsvemitter'
+require_relative 'twb/analysis/datasources/datasourcefilesanalyzer'
+# require_relative 'twb/analysis/datasources/datasourcetablefieldscsvemitter'
+# require_relative 'twb/analysis/datasources/datasourcetablefieldsanalyzer'
 require_relative 'twb/analysis/datasources/categoricalcolorcodinganalyzer'
 require_relative 'twb/analysis/datasources/googlesheetdatasourcesanalyzer'
 require_relative 'twb/analysis/datasources/parametersanalyzer'
 require_relative 'twb/analysis/datasources/datasourceoriginsanalyzer'
+require_relative 'twb/analysis/datasources/fieldsaliasesanalyzer'
+require_relative 'twb/analysis/datasources/extractsanalyzer'
+require_relative 'twb/analysis/sheets/worksheetsummarizer'
 require_relative 'twb/analysis/sheets/worksheetdatastructurecsvemitter'
 require_relative 'twb/analysis/sheets/sheetfiltersanalyzer'
 require_relative 'twb/analysis/sheets/sheetfieldsanalyzer'
@@ -74,5 +80,5 @@ require_relative 'twb/analysis/sheets/sheetsintooltipanalyzer'
 # Represents Tableau Workbooks, their contents, and classes that analyze and manipulate them.
 #
 module Twb
-  VERSION = '4.7.1'
+  VERSION = '4.9.0'
 end

data/lib/twb/analysis/calculatedfields/fieldsaliasesanalyzer.rb

@@ -0,0 +1,110 @@
+#  fieldsaliasesanalyzer.rb - this Ruby script Copyright 2017, 2018 Christopher Gerrard
+#
+#  This program is free software: you can redistribute it and/or modify
+#  it under the terms of the GNU General Public License as published by
+#  the Free Software Foundation, either version 3 of the License, or
+#  (at your option) any later version.
+#
+#  This program is distributed in the hope that it will be useful,
+#  but WITHOUT ANY WARRANTY; without even the implied warranty of
+#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#  GNU General Public License for more details.
+#
+#  You should have received a copy of the GNU General Public License
+#  along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+require 'csv'
+
+module Twb
+module Analysis
+
+  class FieldsAliasesAnalyzer
+
+      include TabTool
+
+      attr_reader   :calculatedFieldsCount, :referencedFieldsCount, :metrics
+
+      @@csvFileName   = 'FieldAliases.csv'
+      @@csvFileHeader = [
+                          'Record #',
+                          "Record Number",
+                          "Workbook",
+                          "Data Source",
+                          'Data Source Caption',
+                          'Data Source (tech)',
+                          'Field',
+                          'Field Caption',
+                          'Field Name (tech)',
+                          'Data Type',
+                          'Role',
+                          'Type',
+                          'Field Value',
+                          'Field Alias'
+                       ]
+
+
+      def initialize(**args)
+          emit "initialize FieldsAliasesAnalyzer args #{args}"
+          @args      = args
+          @recordDir = !@args.nil? && @args[:recordDir] == true
+          @ttdocdir  = @args[:ttdocdir]
+          @csvAdd    = @args[:csvMode] == :add
+          @csvMode   = @csvAdd ? 'a' : 'w'
+          init
+          @funcdoc = {:class=>self.class, :blurb=>"Analyze Fields' Aliases", :description=>'Understanding the aliases for field values can be important and useful.',}
+          #-- CSV records collectors
+          # @csvFormulaFields    = Set.new
+          # @csvFormulaLines     = Set.new
+          #-- Counters setup --
+          # @twbCount              = 0
+          @dataSourcesCount   = 0
+          @aliasedFieldsCount = 0
+          @aliasessCount      = 0
+          #--
+          # @referencedFields     = SortedSet.new
+          #--
+          twbdirLabel = @recordDir.nil? ? nil : 'Workbook Dir'
+          @csvFile   = initCSV(@@calcFieldsCSVFileName, 'Calculated fields and their formulas.',                      @@csvFileHeader )
+          #--
+          @localEmit  = false
+          # @imageFiles = Array.new
+          #--
+          # @doGraph = config(:dograph)
+      end
+
+      def processTWB twb
+        twb.datasources.each do |ds|
+          processDS ds
+        end
+      end
+
+      def processDS ds
+        @dsUIName  = ds.uiname
+        @dsCaption = ds.caption
+        @dsName    = ds.name
+        #--
+        aliasesNodes = ds.node.xpath('.//aliases')
+        aliasesNodes.each do |an|
+          processAliases an
+        end
+      end
+
+      def processAliases aliasesNode
+        #-- Field       --
+        caption         = d.xpath('../../@caption').text
+        techName        = d.xpath('../../@name').text
+        name            = if caption == '' then getName(techName) else caption end
+        dataType        = d.xpath('../../@datatype').text
+        role            = d.xpath('../../@role').text
+        type            = d.xpath('../../@type').text
+
+        #-- Alias       --
+        aliasKey        = d.xpath('./@key').text
+        aliasValue      = d.xpath('./@value').text
+
+
+
+
+
+
+  end  # class FieldsAliasesAnalyzer

data/lib/twb/analysis/datasources/datasourcefilesanalyzer.rb

@@ -0,0 +1,83 @@
+#  datasourcefilesanalyzer.rb - this Ruby script Copyright 2017, 2018 Christopher Gerrard
+#
+#  This program is free software: you can redistribute it and/or modify
+#  it under the terms of the GNU General Public License as published by
+#  the Free Software Foundation, either version 3 of the License, or
+#  (at your option) any later version.
+#
+#  This program is distributed in the hope that it will be useful,
+#  but WITHOUT ANY WARRANTY; without even the implied warranty of
+#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#  GNU General Public License for more details.
+#
+#  You should have received a copy of the GNU General Public License
+#  along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+require 'csv'
+
+module Twb
+module Analysis
+module DataSources
+
+  class DataSourceFilesAnalyzer
+
+      include TabTool
+
+      attr_reader   :filesCount
+
+      @@csvFileName   = 'DataSourceFiles.csv'
+      @@csvFileHeader = [ 'Record #',
+                          'Workbook',
+                          'Workbook Dir',
+                          'Data Source', 
+                          'File Name', 
+                          'File Dir',
+                          'File Modified',
+                          'File Size'
+                        ]
+
+      def initialize(**args)
+          emit "initialize DataSourceFilesAnalyzer args #{args}"
+          @args      = args
+          @recordDir = !@args.nil? && @args[:recordDir] == true
+          @ttdocdir  = @args[:ttdocdir]
+          @csvAdd    = @args[:csvMode] == :add
+          @csvMode   = @csvAdd ? 'a' : 'w'
+          init
+          @funcdoc = {:class=>self.class, :blurb=>"Analyze Data Sources' Files", :description=>'Identifying the files accessed by Data Sources.',}
+          @dataSourcesCount   = 0
+          @recNum             = 0
+          #--
+          @csvFile   = initCSV(@@csvFileName, 'Data Sources and the Files they access.',                      @@csvFileHeader )
+          #--
+          @localEmit  = false
+      end
+
+      def processTWB twb
+        @twbName = twb.name
+        @twDir   = twb.dir
+        twb.datasources.each do |ds|
+          processFiles ds
+        end
+      end
+
+      def processFiles ds
+        dirFiles = Hash.new { |d,fs| d[fs] = Set.new }
+        ds.node.xpath('.//connection[@filename]').each do |fnode|
+            twbFileDir  = fnode['directory']
+            twbFileName = fnode['filename']
+            fqFileName  = twbFileDir.nil? ?  twbFileName : twbFileDir + '/' + twbFileName
+            fileName = File.basename fqFileName
+            fileDir  = File.dirname  fqFileName
+            modtime  = File.mtime    fqFileName
+            size     = File.size     fqFileName
+            data = [$recNum+=1, twbName, twb.dir, ds.uiname, fileName, fileDir, modtime, size]
+            @csvFile  << data
+        end
+      end
+
+  end  # class DataSourceFilesAnalyzer
+
+end # module DataSources
+end # module Analysis
+end # module Twb

data/lib/twb/analysis/datasources/datasourcetablefieldsanalyzer.rb

@@ -0,0 +1,117 @@
+#  datasourcetablefieldsanalyzer.rb - this Ruby script Copyright 2019 Christopher Gerrard
+#
+#  This program is free software: you can redistribute it and/or modify
+#  it under the terms of the GNU General Public License as published by
+#  the Free Software Foundation, either version 3 of the License, or
+#  (at your option) any later version.
+#
+#  This program is distributed in the hope that it will be useful,
+#  but WITHOUT ANY WARRANTY; without even the implied warranty of
+#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#  GNU General Public License for more details.
+#
+#  You should have received a copy of the GNU General Public License
+#  along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+require 'twb'
+
+module Twb
+module Analysis
+module DataSources
+
+  class require_relative DataSourceTableFieldsAnalyzer
+
+    attr_reader :csvFileName, :csvRecords
+    attr_reader :dsCount,     :tablesCount, :fieldsCount  
+
+    @@csvFileName        = 'DataSourceTableFields.csv'
+    @csvFileDescription = 'Contains CSV records, each containing the details for an individual Field contained in a table in the Data Source. Data Sources are identified within their host Workbook. '
+
+    @@csvHeader = ['Record #',
+                   'Workbook',           
+                   'Workbook Dir',
+                   'Data Source',
+                   'Data Source (tech)',
+                   'Table',
+                   'Field',
+                   'Field (code)',
+                   'Field (table)',
+                  ]
+
+
+    def initialize
+      @csvFile = CSV.open(@@csvFileName, 'w')
+      @csvFile << @@csvHeader
+      @outputs = Set.new
+      @outputs << @@csvFileName
+      @dsCount      = 0
+      @tablesCount  = 0
+      @fieldsCount  = 0
+      @csvRecords   = Set.new
+    end
+
+    def doc
+      {
+        :filedoc => [
+                     { :file        => @@csvFileName,
+                       :description => @csvFileDescription,
+                       :header      => @@csvHeader 
+                     }
+                    ],
+        :outputs => @outputs
+      }
+    end
+
+    def self.csvHeader
+      @@csvHeader
+    end
+
+    def self.csvFileType
+      @@csvFileName
+    end
+
+    def outputs
+      @outputs
+    end
+
+    def processTwb twb
+      @recNum = 0
+      @twb    = twb if twb.instance_of? Twb::Workbook
+      @twb    = Twb::Workbook.new(twb) if twb.instance_of? String
+      # --
+      dss = @twb.datasources
+      dss.each do |ds|
+        tables = Set.new
+        @dsCount += 1
+        ds.node.xpath('./connection/cols/map').each do |cnode|
+          # puts cnode
+          key         = cnode.attribute('key').text
+          codename    = key.gsub(/^\[|\]$/,'')
+          fielduiname = ds.fieldUIName(codename)
+          value       = cnode.attribute('value').text.gsub(/^\[|\]$/,'')
+          parts       = value.split('].[')
+          # puts "%-30s  %-45s  %s \n " % [fielduiname, value, parts]
+          csvRecord = [ @fieldsCount += 1,
+                        @twb.name,
+                        @twb.dir,
+                        ds.uiname,
+                        ds.name,
+                        parts[0],
+                        fielduiname,
+                        codename,
+                        parts[1] 
+                      ]
+          tables.add parts[0]
+          @csvRecords.add csvRecord
+          @csvFile << csvRecord
+        end
+        @tablesCount += tables.length
+      end
+    end
+
+  end # class CSVEmitter
+
+
+end   # module DataSources
+end   # module Analysis
+end   # module Twb

data/lib/twb/analysis/datasources/extractsanalyzer.rb

@@ -0,0 +1,199 @@
+#  worksheetsummarizer.rb Copyright (C) 2018  Chris Gerrard
+#
+#  This program is free software: you can redistribute it and/or modify
+#  it under the terms of the GNU General Public License as published by
+#  the Free Software Foundation, either version 3 of the License, or
+#  (at your option) any later version.  
+#
+#  This program is distributed in the hope that it will be useful,
+#  but WITHOUT ANY WARRANTY; without even the implied warranty of
+#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#  GNU General Public License for more details.
+#
+#  You should have received a copy of the GNU General Public License
+#  along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+require 'twb'
+require 'csv'
+
+module Twb
+module Analysis
+module DataSources
+
+  class ExtractsAnalyzer
+
+    include TabTool
+
+    attr_accessor :localEmit
+
+    def initialize(**args)
+      @args      = args
+      @recordDir = !@args.nil? && @args[:recordDir] == true
+      @ttdocdir  = @args[:ttdocdir]
+      @csvAdd    = !@args.nil? && args[:csvMode] == :add
+      @csvMode   = @csvAdd ? 'a' : 'w'
+      init
+      @funcdoc   = {:class=>self.class, :blurb=>'Analyze Extracts', :description=>'Identifies Data Source Extracts & records relevant data.',}
+      #--
+      docFileName  = docFile('Extracts.csv')
+      @extractsCSV = CSV.open(docFileName,@csvMode)
+      #--
+        # <extract count='-1' enabled='true' units='records'>
+        #   <connection access_mode='readonly' 
+        #               authentication='auth-none' 
+        #               author-locale='en_US' 
+        #               class='hyper' 
+        #               dbname='C:/tech/Tableau/Tableau Tools/Ruby/gems/twb/lib/twb/analysis/datasources/data/Sample - World Bank Indicators (Excel).hyper' 
+        #               default-settings='yes' 
+        #               schema='Extract' 
+        #               sslmode='' 
+        #               update-time='05/17/2019 10:39:21 PM' 
+        #               username='tableau_internal_user'>
+        #     <relation name='Extract' table='[Extract].[Extract]' type='table' />
+        #     <refresh>
+        #       <refresh-event add-from-file-path='Sample - World Bank Indicators (Excel)' increment-value='%null%' refresh-type='create' rows-inserted='2354' timestamp-start='2019-05-17 22:39:21.474' />
+        #     </refresh>
+        #   </connection>
+        # </extract>
+        #--
+      unless @csvAdd
+        @csvHeader = [ 'Rec #', 
+                       'Workbook', 
+                       'Data Source',
+                       'Enabled',
+                       'Units',
+                       'Access Mode',
+                       'Authentication',
+                       'Class',
+                       'Extract Name (FQ)',
+                       'Extract Name',
+                       'Extract Directory',
+                       'Default Settings',
+                       'Schema',
+                       'SSL Mode',
+                       'Updated (UTC)',
+                       'User Name'
+                     ]
+        if @recordDir
+          @csvHeader.push 'Workbook Dir'
+        end
+        @extractsCSV << @csvHeader
+      end
+      addDocFile @extractsCSV, docFileName, "Workbooks and their Data Source Extracts."
+      #--
+      @twbCount     = 0
+      @dsCount      = 0
+      @extractCount = 0
+      @recNum       = 0
+    end
+
+    def metrics
+      {
+        '# of Data Sources' => @dsCount,
+        '# of Extracts'     => @extractCount
+      }
+    end
+
+    def processTWB twb
+       @twb     = twb
+       @twbName = @twb.name
+       @twbDir  = @twb.dir
+       emit "   -- #{@twbName}"
+       @twbCount += 1
+       parseDataSources
+       finis
+    end
+
+    def parseDataSources
+      @dataSources = @twb.datasources
+      @dataSources.each do |ds|
+        @dsCount += 1
+        emit "DATA SOURCE:: #{ds.name}"
+        dsNode = ds.node
+        exnode = ds.node.at_xpath('./extract')
+        unless exnode.nil?
+          cnNode = exnode.at_xpath('./connection')
+          # emit true, cnNode
+          unless cnNode.nil?
+            @extractCount += 1
+            dbName = cnNode['dbname']
+            # emit true, "cnNode['dbname'] :: #{cnNode['dbname']}"
+            # unless dbName.nil?
+              exName = File.basename dbName
+              exDir  = File.dirname  dbName
+            # end
+            recordCSV [ @twbName,
+                        ds.uiname,
+                        exnode['enabled'],
+                        exnode['units'],
+                        cnNode['access_mode'],
+                        cnNode['authentication'],
+                        cnNode['class'],
+                        dbName,
+                        exName,
+                        exDir,
+                        cnNode['default-settings'],
+                        cnNode['schema'],
+                        cnNode['sslmode'],
+                        cnNode['update-time'],
+                        cnNode['username']
+                      ]
+          end
+        end
+      end
+    end
+
+        # <extract count='-1' enabled='true' units='records'>
+        #   <connection access_mode='readonly' 
+        #               authentication='auth-none' 
+        #               author-locale='en_US' 
+        #               class='hyper' 
+        #               dbname='C:/tech/Tableau/Tableau Tools/Ruby/gems/twb/lib/twb/analysis/datasources/data/Sample - World Bank Indicators (Excel).hyper' 
+        #               default-settings='yes' 
+        #               schema='Extract' 
+        #               sslmode='' 
+        #               update-time='05/17/2019 10:39:21 PM' 
+        #               username='tableau_internal_user'>
+        #     <relation name='Extract' table='[Extract].[Extract]' type='table' />
+        #     <refresh>
+        #       <refresh-event add-from-file-path='Sample - World Bank Indicators (Excel)' increment-value='%null%' refresh-type='create' rows-inserted='2354' timestamp-start='2019-05-17 22:39:21.474' />
+        #     </refresh>
+        #   </connection>
+        # </extract>
+        #--
+      # unless @csvAdd
+      #   @csvHeader = [ 'Rec #', 
+      #                  'Workbook', 
+      #                  'Data Source',
+      #                  'Enabled',
+      #                  'Units',
+      #                  'Access Mode',
+      #                  'Authentication',
+      #                  'Class',
+      #                  'Db Name',
+      #                  'Default Settings',
+      #                  'Schema',
+      #                  'SSL Mode',
+      #                  'Updated',
+      #                  'User Name'
+      #                ]
+
+
+
+
+    private
+
+      def recordCSV record
+        numberedRec = [@recNum+=1] + record
+        if @recordDir
+          @extractsCSV  << numberedRec.push(@twbDir)
+        else
+          @extractsCSV  << numberedRec
+        end
+      end
+
+  end # class ExtractsAnalyzer
+
+end # module DataSources
+end # module Analysis
+end # module Twb

data/lib/twb/analysis/datasources/fieldsaliasesanalyzer.rb

@@ -0,0 +1,117 @@
+#  fieldsaliasesanalyzer.rb - this Ruby script Copyright 2017, 2018 Christopher Gerrard
+#
+#  This program is free software: you can redistribute it and/or modify
+#  it under the terms of the GNU General Public License as published by
+#  the Free Software Foundation, either version 3 of the License, or
+#  (at your option) any later version.
+#
+#  This program is distributed in the hope that it will be useful,
+#  but WITHOUT ANY WARRANTY; without even the implied warranty of
+#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#  GNU General Public License for more details.
+#
+#  You should have received a copy of the GNU General Public License
+#  along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+require 'csv'
+
+module Twb
+module Analysis
+module DataSources
+
+  class FieldsAliasesAnalyzer
+
+      include TabTool
+
+      attr_reader   :calculatedFieldsCount, :referencedFieldsCount, :metrics
+
+      @@csvFileName   = 'FieldAliases.csv'
+      @@csvFileHeader = [
+                          'Record #',
+                          "Workbook",
+                          "Data Source",
+                          'Field',
+                          'Value - Db',
+                          'Value - Alias'
+                        ]
+
+
+      def initialize(**args)
+          emit "initialize FieldsAliasesAnalyzer args #{args}"
+          @args      = args
+          @recordDir = !@args.nil? && @args[:recordDir] == true
+          @ttdocdir  = @args[:ttdocdir]
+          @csvAdd    = @args[:csvMode] == :add
+          @csvMode   = @csvAdd ? 'a' : 'w'
+          init
+          @funcdoc = {:class=>self.class, :blurb=>"Analyze Fields' Aliases", :description=>'Understanding the aliases for field values can be important and useful.',}
+          #-- CSV records collectors
+          # @csvFormulaFields    = Set.new
+          # @csvFormulaLines     = Set.new
+          #-- Counters setup --
+          # @twbCount              = 0
+          @dataSourcesCount   = 0
+          @aliasedFieldsCount = 0
+          @aliasessCount      = 0
+          @recNum             = 0
+          #--
+          # @referencedFields     = SortedSet.new
+          #--
+          @csvFile   = initCSV(@@csvFileName, 'Fields and their aliased values.',                      @@csvFileHeader )
+          #--
+          @localEmit  = false
+          # @imageFiles = Array.new
+          #--
+          # @doGraph = config(:dograph)
+      end
+
+      def processTWB twb
+        @twbName = twb.name
+        @twDir   = twb.dir
+        twb.datasources.each do |ds|
+          processDS ds
+        end
+      end
+
+      def processDS ds
+        @dsUIName  = ds.uiname
+        @dsCaption = ds.caption
+        @dsName    = ds.name
+        #--
+        ds.columnFields.each do |fld|
+          if fld.hasaliases
+            @fieldName    = fld.name
+            @fieldCaption = fld.caption
+            @fieldUIName  = fld.uiname
+            processAliases(fld) 
+          end
+        end
+      end
+
+      def processAliases field
+        aliasNodes = field.node.xpath('./aliases//alias')
+        aliasNodes.each do |anode|
+          key      = anode['key']
+          value    = anode['value']
+          numberedRec = [
+                          @recNum +=1, 
+                          @twbName,
+                          @dsUIName,
+                          @fieldName,
+                          anode['key'].to_s.gsub(/^["]+/,'').gsub(/["]+$/,''),
+                          anode['value']
+                        ]
+          #--
+          if @recordDir
+            @csvFile  << numberedRec.push(@twbDir)
+          else
+            @csvFile  << numberedRec
+          end
+        end
+      end
+
+  end  # class FieldsAliasesAnalyzer
+
+end # module DataSources
+end # module Analysis
+end # module Twb

data/lib/twb/analysis/sheets/sheetsintooltipanalyzer.rb

@@ -70,10 +70,10 @@ module Analysis
 
     def parseSheets
       @sheets      = @twb.worksheets
-      puts " #Sheets: #{@sheets.length}"
+      # puts " #Sheets: #{@sheets.length}"
       @sheetCount += @sheets.length
       @sheets.each do |sheet|
-        puts "SHEET:: #{sheet.name}"
+        # puts "SHEET:: #{sheet.name}"
         # @dashCount += 1
         ttip = sheet.tooltip
         runs = ttip.xpath('.//run')
@@ -81,12 +81,12 @@ module Analysis
           # puts run.text
           text = run.text
           isViz = text =~ /.*<Sheet[ ]+name=.*>.*/
-          puts "\t   #{text}"
+          # puts "\t   #{text}"
           if isViz
             vizCode = text.gsub(/^[^<]*/,'').gsub(/[^>]*$/,'')
             vizNode = Nokogiri::XML(vizCode).at_xpath('./Sheet')
-            puts "\t   #{vizCode}"
-            puts "\t   #{vizNode.class}  \t #{vizNode}"
+            # puts "\t   #{vizCode}"
+            # puts "\t   #{vizNode.class}  \t #{vizNode}"
             recordCSV [@twbName, sheet.name, vizNode['name'], vizNode['maxwidth'], vizNode['maxheight']]
           end
           

data/lib/twb/analysis/sheets/worksheetsummarizer.rb

@@ -0,0 +1,118 @@
+#  worksheetsummarizer.rb Copyright (C) 2018  Chris Gerrard
+#
+#  This program is free software: you can redistribute it and/or modify
+#  it under the terms of the GNU General Public License as published by
+#  the Free Software Foundation, either version 3 of the License, or
+#  (at your option) any later version.  
+#
+#  This program is distributed in the hope that it will be useful,
+#  but WITHOUT ANY WARRANTY; without even the implied warranty of
+#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#  GNU General Public License for more details.
+#
+#  You should have received a copy of the GNU General Public License
+#  along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+require 'twb'
+require 'csv'
+
+module Twb
+module Analysis
+
+  class WorksheetSummarizer
+
+    include TabTool
+
+    attr_accessor :localEmit
+
+    # Worksheet attributes; from Worksheet.rb
+      # attr_reader   :node,   :name,      :datasourcenames, :datasources
+      # attr_reader   :panesCount
+      # attr_reader   :fields, :rowFields, :colFields, :paneFields, :datasourceFields, :pageFields, :encodedFields, :slicesFields
+      # attr_reader   :filters
+      # attr_reader   :tooltip
+      # attr_accessor :hidden, :visible
+
+    def initialize(**args)
+      @args      = args
+      @recordDir = !@args.nil? && @args[:recordDir] == true
+      @ttdocdir  = @args[:ttdocdir]
+      @csvAdd    = !@args.nil? && args[:csvMode] == :add
+      @csvMode   = @csvAdd ? 'a' : 'w'
+      init
+      @funcdoc   = {:class=>self.class, :blurb=>'Summarize Worksheets', :description=>'Identifies Worksheets & summarizes them.',}
+      #--
+      docFileName     = docFile('WorkheetSummaries.csv')
+      @worksheetsCSV  = CSV.open(docFileName,@csvMode)
+      unless @csvAdd
+        @csvHeader = [ 'Rec #', 
+                       'Workbook', 
+                       'Worksheet',
+                       'Hidden', 'Visible',
+                       '# Data Sources', 
+                       '# Fields - Data', '# Fields - Rows', '# Fields - Cols',
+                       'Tooltip?', 'Filters?'
+                     ]
+        if @recordDir
+          @csvHeader.push 'Workbook Dir'
+        end
+        @worksheetsCSV << @csvHeader
+      end
+      addDocFile @worksheetsCSV, docFileName, "Workbooks and their Worksheets' summaries."
+      #--
+      @twbCount    = 0
+      @sheetCount  = 0
+      @recNum      = 0
+    end
+
+    def metrics
+      {
+        '# of Worksheets' => @sheetCount
+      }
+    end
+
+    def processTWB twb
+       @twb     = twb
+       @twbName = @twb.name
+       @twbDir  = @twb.dir
+       @modTime = @twb.modtime
+       emit "   -- #{@twbName}"
+       @twbCount += 1
+       parseSheets
+       finis
+    end
+
+    def parseSheets
+      @worksheets  = @twb.worksheets
+      @worksheets.each do |sheet|
+        emit "SHEET:: #{sheet.name}"
+        @sheetCount += 1
+        recordCSV [ @twbName, 
+                    sheet.name, 
+                    sheet.hidden, 
+                    sheet.visible,
+                    sheet.datasources.length,
+                    sheet.datasourceFields.length,
+                    sheet.rowFields.length,
+                    sheet.colFields.length,
+                    !sheet.tooltip.nil?,
+                    !sheet.filters.nil?,
+                  ]
+      end
+    end
+
+    private
+
+      def recordCSV record
+        numberedRec = [@recNum+=1] + record
+        if @recordDir
+          @worksheetsCSV  << numberedRec.push(@twbDir)
+        else
+          @worksheetsCSV  << numberedRec
+        end
+      end
+
+  end #class WorksheetSummarizer
+
+end # module Analysis
+end # module Twb

data/lib/twb/columnfield.rb

@@ -58,7 +58,8 @@ module Twb
     # visual-totals
 
     attr_reader :node,       :properties
-    attr_reader :name,       :caption,       :uiname,   :alias
+    attr_reader :name,       :caption,       :uiname   
+    attr_reader :hasaliases, :alias
     attr_reader :dataType,   :defaultFormat, :paramDomainType
     attr_reader :role,       :type,          :value
     attr_reader :alias,      :semanticRole,  :aggregation
@@ -109,7 +110,7 @@ module Twb
       return role
     end
 
-    def  calcField
+    def calcField
       @calcField ||= loadCalcField
     end
 
@@ -137,6 +138,25 @@ module Twb
       @commentLines ||= loadComment
     end
 
+    def hasaliases
+      @hasaliases ||= !@node.at_xpath('./aliases').nil?
+    end
+
+    def aliases
+      @aliases ||= loadAliases
+    end
+
+    def loadAliases
+      @aliases = {}
+      if hasaliases
+        @node.at_xpath('./aliases//alias').each do |an|
+          dbvalue = an['key'].to_s.gsub(/^["]+/,'').gsub(/["]+$/,'')
+          @aliases[ dbvalue ] = an['value']
+        end
+      end
+      return @aliases
+    end
+
     # COMMENTED FIELDS
     # --------------------------------------
     # <column caption='Calculation1' datatype='real' name='[Calculation_821344020759588865]' role='measure' type='quantitative'>

data/lib/twb/datasource.rb

@@ -235,8 +235,14 @@ module Twb
       'Parameters'.eql? @name 
     end
 
+    def columnField fieldName
+      loadColumnFields if @columnFieldsMap.nil?
+      @columnFieldsMap[ fieldName ]
+    end
+
+
     def columnFields
-      @columnFields ||= @columnFields = loadColumnFields
+      @columnFields ||= loadColumnFields
     end
 
     def columnFieldsMap

data/lib/twb/tabtool.rb

@@ -35,7 +35,7 @@ module TabTool
 
   # @logfilename = 'TabTools.ttlog'
   @@loglevel    = Logger::DEBUG #INFO
-  # @localEmit   = false
+  @localEmit    = false
   # @docDirSet   = false
 
   def init

data/lib/twb/util/atomizecsvrecords.rb

@@ -0,0 +1,105 @@
+#  calculatedfieldsanalyzer.rb - this Ruby script Copyright 2017, 2018 Christopher Gerrard
+#
+#  This program is free software: you can redistribute it and/or modify
+#  it under the terms of the GNU General Public License as published by
+#  the Free Software Foundation, either version 3 of the License, or
+#  (at your option) any later version.
+#
+#  This program is distributed in the hope that it will be useful,
+#  but WITHOUT ANY WARRANTY; without even the implied warranty of
+#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#  GNU General Public License for more details.
+#
+#  You should have received a copy of the GNU General Public License
+#  along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+# require 'nokogiri'
+# require 'twb'
+# require 'set'
+require 'csv'
+
+module Twb
+module Util
+
+  class AtomizeCSVRecords
+
+      include TabTool
+      # include Graph
+
+      attr_reader   :metrics
+      attr_accessor :ttdocdir
+
+      @@ttlogfile   = 'AtomizeCSVRecords.ttlog'
+      @@csvFileName = 'AtomizedCSVRecords.csv'  
+
+      def initialize(**args)
+          @ttdocdir  = @args[:ttdocdir]
+          @csvAdd    = @args[:csvMode] == :add
+          @csvMode   = @csvAdd ? 'a' : 'w'
+          init
+          @funcdoc   = {:class=>self.class, :blurb=>'Analyze Calculated Fields', :description=>'Calculated fields can be complex, this tool provides robust coverage.',}
+          # --
+          @executed  = false
+      end
+
+      def loadFields fileName
+        fields = Set.new
+        file = File.open(fileName)
+        unless file.nil?
+          file.each_line { |line|
+            fields.add line.chomp
+          }
+        end
+        return fields
+      end
+
+      def processTWB workbook
+        puts "\n\t ALERT: Utility tool: #{self.class} - does not implement Workbook processing"
+        puts   "\t                                    - deferring to single execution"
+        processData fileName
+      end
+
+      def processData fileName
+        unless @executed
+          emit true, "initialize AtomizeCSVRecords args #{args}"
+          @atomicCSV = CSV.open(@@csvFileName, 'w')
+          # --
+          inputFileName = fileName.nil? ? 'csvData.csv' : fileName
+          keyFields     = loadFields 'key.fields'
+          excludeFields = loadFields 'exclude.fields'
+          puts "\t Using these key fields: #{$keyFields.inspect}"
+          puts "\t Excluding these fields: #{$excludeFields.inspect}"
+
+            $fieldValsCSV = '.csv'
+            $atomicCSV    = CSV.open($fieldValsCSV, 'w')
+            $atomicFields = $keyFields.to_a + [ 'Rec #', 'Field', 'Value' ]
+            puts "\n\t CSV output fields: #{$atomicFields.inspect}"
+            $atomicCSV << $atomicFields
+
+
+            $recCnt, $fldsCnt = 0, 0
+
+            $recNum = 0
+
+          @executed = true
+        end
+      end
+
+      def metrics
+        @metrics ||= loadMetrics
+      end
+
+      def loadMetrics
+        @metrics = {
+                     '# of Data Sources'      => @dataSourcesCount,
+                     '# of Calculated Fields' => @calculatedFieldsCount,
+                     '# of Referenced Fields' => @referencedFieldsCount,
+                   }
+      end
+
+      #-- private methods begin here, to end of class
+
+  end #  class
+
+end   # module Analysis
+end   # module Twb

data/lib/twb/util/csvrecordsatomizer.rb

@@ -0,0 +1,114 @@
+#  CSVRecordsAtomizer.rb - this Ruby script Copyright 2019 Christopher Gerrard
+#
+#  This program is free software: you can redistribute it and/or modify
+#  it under the terms of the GNU General Public License as published by
+#  the Free Software Foundation, either version 3 of the License, or
+#  (at your option) any later version.
+#
+#  This program is distributed in the hope that it will be useful,
+#  but WITHOUT ANY WARRANTY; without even the implied warranty of
+#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#  GNU General Public License for more details.
+#
+#  You should have received a copy of the GNU General Public License
+#  along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+require 'csv'
+
+module Twb
+module Util
+
+  class CSVRecordsAtomizer
+
+      include TabTool
+
+      attr_reader   :metrics
+      attr_accessor :ttdocdir
+
+      @@ttlogfile    = 'AtomizeCSVRecords.ttlog'
+      @@csvFileName  = 'AtomizedCSVRecords.csv'
+      @@atomFields   = ['Record #', 'Field Name', 'Field Value'] 
+
+      def initialize(**args)
+          @args      = args
+          @ttdocdir  = @args[:ttdocdir]
+          @csvAdd    = @args[:csvMode] == :add
+          @csvMode   = @csvAdd ? 'a' : 'w'
+          init
+          @funcdoc   = {:class=>self.class, :blurb=>'Analyze Calculated Fields', :description=>'Calculated fields can be complex, this tool provides robust coverage.'}
+          # --
+          @executed  = false
+      end
+
+      def processFile fileName
+          emit true, "#{self.class} - processFile: #{fileName} :: #{fileName.class}"
+            # --
+          srcFields = Set.new CSV.open(fileName, 'r:bom|utf-8', &:readline)
+          emit true, "\t Inspecting these #{srcFields.length} #{srcFields.class} fields: "
+          keyFields     = loadFields(fileName, :keyfields)
+          excludeFields = loadFields(fileName, :excludefields)
+          recordFields  = (srcFields - keyFields - excludeFields).to_a
+          emit true, "\t       Using these key fields: #{keyFields.inspect}"
+          emit true, "\t       Excluding these fields: #{excludeFields.inspect}"
+          emit true, "\t       Recording these fields: #{recordFields.inspect}"
+          # --
+          atomizedFields  = keyFields.to_a + @@atomFields
+          dataBaseName    = File.basename(fileName, ".*")
+          atomizedCSVFile = "#{dataBaseName}.Atomized.csv" 
+          atomizedCSVFile = CSV.open(atomizedCSVFile,'w') 
+          atomizedCSVFile << keyFields.to_a + @@atomFields
+          # --
+          @recNum = 0
+          CSV.open(fileName, 'r:bom|utf-8', headers: true) do |csv|
+            csv.each do |row|
+              @recNum += 1
+              atomRec = []
+              keyFields.each do |kf|
+                atomRec << row.fetch(kf)
+              end
+              atomRec << @recNum
+              recordFields.each do |rf|
+                atomizedCSVFile << atomRec + [rf, row.fetch(rf)] 
+              end
+            end
+          end
+      end
+
+      private
+
+      def loadFields fileName, type
+        puts "\n\n\t loadFields -#{fileName}-  -#{type}-"
+        fields    = Set.new
+        auxFile   = "#{fileName}.#{type}"
+        isAuxFile = File.exist?(auxFile)
+        emit true, "\t\t Loading auxilary data: %-12s %7s %-s" % [type, isAuxFile, auxFile]
+        if isAuxFile
+          # file = File.open(fileName)
+          File.open(auxFile).each_line { |line|
+            fields.add line.chomp
+          }
+        end
+        return fields.to_a
+      end
+
+      def processData fileName
+      end
+
+      def metrics
+        @metrics ||= loadMetrics
+      end
+
+      def loadMetrics
+        @metrics = {
+                     '# of Data Sources'      => @dataSourcesCount,
+                     '# of Calculated Fields' => @calculatedFieldsCount,
+                     '# of Referenced Fields' => @referencedFieldsCount,
+                   }
+      end
+
+      #-- private methods begin here, to end of class
+
+  end #  class
+
+end   # module Analysis
+end   # module Twb

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: twb
 version: !ruby/object:Gem::Version
-  version: 4.7.1
+  version: 4.9.0
 platform: ruby
 authors:
 - Chris Gerrard
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2019-03-22 00:00:00.000000000 Z
+date: 2019-09-23 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: creek
@@ -63,15 +63,20 @@ files:
 - lib/twb/analysis/annotatedfieldscsvemitter.rb
 - lib/twb/analysis/calculatedfields/calculatedfieldsanalyzer.rb
 - lib/twb/analysis/calculatedfields/csvemitter.rb
+- lib/twb/analysis/calculatedfields/fieldsaliasesanalyzer.rb
 - lib/twb/analysis/calculatedfields/groupfieldsanalyzer.rb
 - lib/twb/analysis/calculatedfields/markdownemitter.rb
 - lib/twb/analysis/datasources/categoricalcolorcodinganalyzer.rb
 - lib/twb/analysis/datasources/datasourcefieldsanalyzer.rb
 - lib/twb/analysis/datasources/datasourcefieldscsvemitter.rb
+- lib/twb/analysis/datasources/datasourcefilesanalyzer.rb
 - lib/twb/analysis/datasources/datasourceoriginsanalyzer.rb
 - lib/twb/analysis/datasources/datasourcesenumerator.rb
 - lib/twb/analysis/datasources/datasourceslocationsanalyzer.rb
+- lib/twb/analysis/datasources/datasourcetablefieldsanalyzer.rb
 - lib/twb/analysis/datasources/datasourcetablefieldscsvemitter.rb
+- lib/twb/analysis/datasources/extractsanalyzer.rb
+- lib/twb/analysis/datasources/fieldsaliasesanalyzer.rb
 - lib/twb/analysis/datasources/googlesheetdatasourcesanalyzer.rb
 - lib/twb/analysis/datasources/parametersanalyzer.rb
 - lib/twb/analysis/documentedfieldscsvemitter.rb
@@ -84,6 +89,7 @@ files:
 - lib/twb/analysis/sheets/sheetsintooltipanalyzer.rb
 - lib/twb/analysis/sheets/sheetsourcesanalyzer.rb
 - lib/twb/analysis/sheets/worksheetdatastructurecsvemitter.rb
+- lib/twb/analysis/sheets/worksheetsummarizer.rb
 - lib/twb/analysis/workbooksummaryanalyzer.rb
 - lib/twb/calculatedfield.rb
 - lib/twb/codedfield.rb
@@ -118,6 +124,8 @@ files:
 - lib/twb/there.rb
 - lib/twb/twbcodedfield.rb
 - lib/twb/twbtest.rb
+- lib/twb/util/atomizecsvrecords.rb
+- lib/twb/util/csvrecordsatomizer.rb
 - lib/twb/util/cypher.rb
 - lib/twb/util/cypherpython.rb
 - lib/twb/util/docprep.rb