twb 2.2.1 → 3.7.2

data/lib/twb/analysis/CalculatedFields/MarkdownEmitter.rb

@@ -1,4 +1,4 @@
-#  calculatedfieldsanalyzer.rb - this Ruby script Copyright 2017 Christopher Gerrard
+#  calculatedfieldsanalyzer.rb - this Ruby script Copyright 2017, 2018 Christopher Gerrard
 #
 #  This program is free software: you can redistribute it and/or modify
 #  it under the terms of the GNU General Public License as published by
@@ -13,7 +13,7 @@
 #  You should have received a copy of the GNU General Public License
 #  along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
-require 'twb'
+# require 'twb'
 
 module Twb
 module Analysis
@@ -24,46 +24,60 @@ module CalculatedFields
     attr_reader :docFileName
 
     def initialize
-      # puts ""
+      #@docFile.puts ""
     end
 
     def processTwb twb
-      twb = File.basename(twb)
-      @twb = Twb::Workbook.new twb
-      @docFileName = twb + '.CalculatedFields.md'
+      # twb = File.basename(twb)
+      @twb = twb #Twb::Workbook.new twb
+      @docFileName = './ttdoc/' + @twb.name + '.CalculatedFields.md'
       @docFile     = File.open(@docFileName,'w')
-      @docFile.puts "## #{twb}"
+      @docFile.puts "## #{twb.name}"
       dsNames = @twb.datasourceUINames
       @docFile.puts "#{dsNames.length} Data Sources"
-      dsNames.each do |dsn|
-        ds = @twb.datasource dsn
-      #   puts "  => #{ds.uiname} "
-      # end
-      # dss = @twb.datasources
-      # dss.each do |ds|
-        @docFile.puts "### #{ds.uiname}"
-        calcFields = ds.calculatedFieldsMap.sort_by { |fldName,calc| fldName }
-        calcFields.each do |fldname, field|
-          calculation = field.calculation
-          @docFile.puts "\n##### #{fldname} "
-          @docFile.puts "###### ...description..."
-          @docFile.puts "```"
-          if calculation.has_formula 
-            @docFile.puts calculation.formulaResolved
-            @docFile.puts "\n -- Fields  --" unless calculation.calcFields.empty?
-            refFields = SortedSet.new
-            calculation.calcFields.each do |cf|
-              fds =  if cf.dataSourceRef == :remote then "<<#{cf.dataSource}>>" else '' end
-              refFields.add "#{cf.uiName} \t #{fds}"
-            end
-            refFields.each do |rf|
-              @docFile.puts "    #{rf}"
+      
+
+      @twb.datasources.each do |ds|
+        @docFile.puts "# #{ds.uiname}"
+        @docFile.puts "__has #{ds.calculatedFields.length} calculated fields__\n "
+        cnt = 0
+        ds.calculatedFields.each do |cf|
+          cnt += 1
+           @docFile.puts "### '#{cf.uiname}'"
+           @docFile.puts "```"
+            cf.formulaResolvedLines.each do 
+                |l|@docFile.puts "#{l.gsub('<<','[').gsub('>>',']')}" 
             end
-            @docFile.puts "```"
-          end
+           @docFile.puts "```"
+           if cf.calcFields.length > 0
+               fieldsRefOrder  = []
+               fieldsSortSet   = SortedSet.new
+               cf.calcFields.each do |field|
+                   fieldsRefOrder.push field.uiname
+                   fieldsSortSet << field.uiname
+               end
+               if fieldsRefOrder != fieldsSortSet.to_a
+                  @docFile.puts "```"
+                  @docFile.puts "Fields - reference order:"
+                   fieldsRefOrder.each do |field|
+                      @docFile.puts "    '#{field}'"
+                   end
+                  @docFile.puts "```"
+               end
+               if fieldsSortSet.length > 0
+                  @docFile.puts "```"
+                  @docFile.puts "Fields:"
+                   fieldsSortSet.each do |field|
+                      @docFile.puts "    '#{field}'"
+                   end
+                  @docFile.puts "```"
+               end
+           end
+           @docFile.puts "\n "
         end
-      end
-    end
+        @docFile.puts "counted #{cnt} calculated fields\n "
+      end  # twb.datasources.each
+    end    # def processTwb twb
 
   end # class MarkdownEmitter
 

data/lib/twb/analysis/DataSources/DataSourceFieldsCSVEmitter.rb

@@ -1,4 +1,4 @@
-#  calculatedfieldsanalyzer.rb - this Ruby script Copyright 2017 Christopher Gerrard
+#  calculatedfieldsanalyzer.rb - this Ruby script Copyright 2017, 2018 Christopher Gerrard
 #
 #  This program is free software: you can redistribute it and/or modify
 #  it under the terms of the GNU General Public License as published by
@@ -25,35 +25,24 @@ module DataSources
     attr_reader :csvFileName, :csvRecords
     attr_reader :dsCount,     :fieldsCount  
 
-    @@csvHeader = ['Record #',
-                   'Workbook',           
-                   'Workbook Dir',
-                   'Data Source',
-                   'Field',
-                   'Twb Type',
-                   'Hidden',            # props[:hidden],
-                   'Type - Column',     # props[:columnField],
-                   'Type - Calculated', # props[:calculatedField],
-                   'Type - Db',         # props[:dbField],
-                   'Type - Mapped',     # props[:mappedField] 
-                   'Type - MetaData',     # props[:mappedField]  
-                  ]
-    @@csvFileType = 'TwbDataSourceFields'
-    @@csvFileName = @@csvFileType + '.csv'
-
-    @@tallHeader = ['Field #',
-                    'Workbook',           
-                    'Workbook Dir',
-                    'Data Source',
-                    'Field',
-                    'Field Type',
-                    'Property - Name',
-                    'Property - Value'
-                   ]
+    # @@csvHeader = ['Record #',
+    #                'Workbook',           
+    #                'Workbook Dir',
+    #                'Data Source',
+    #                'Field',
+    #                'Twb Type',
+    #                'Hidden',            # props[:hidden],
+    #                'Type - Column',     # props[:columnField],
+    #                'Type - Calculated', # props[:calculatedField],
+    #                'Type - Db',         # props[:dbField],
+    #                'Type - Mapped',     # props[:mappedField] 
+    #                'Type - MetaData',     # props[:mappedField]  
+    #               ]
+    # @@csvFileType = 'TwbDataSourceFields'
+    # @@csvFileName = @@csvFileType + '.csv'
+
     @@fullFileType = 'TwbDataSourceFieldsDetails'
     @@fullFileName = @@fullFileType + '.csv'
-
-
     @@fullHeader = ['Field #',
                     'Workbook',           
                     'Workbook Dir',
@@ -65,22 +54,32 @@ module DataSources
                     'Property - Name',
                     'Property - Value'
                    ]
-    @@tallFileType = 'TwbDataSourceFieldsTech'
-    @@tallFileName = @@tallFileType + '.csv'
+
+    # @@techFileType = 'TwbDataSourceFieldsTech'
+    # @@techFileName = @@techFileType + '.csv'
+    # @@techHeader = ['Field #',
+    #                 'Workbook',           
+    #                 'Workbook Dir',
+    #                 'Data Source',
+    #                 'Field',
+    #                 'Field Type',
+    #                 'Property - Name',
+    #                 'Property - Value'
+    #                ]
 
     def initialize
-      @csvFile     = CSV.open(@@csvFileName,'w')
-      @csvFile     << @@csvHeader
-      @csvRecords  = Set.new
-      puts "Opened: #{!@csvFile.nil?} - #{@@csvFileName}"
+      # @csvFile     = CSV.open(@@csvFileName,'w')
+      # @csvFile     << @@csvHeader
+      # @csvRecords  = Set.new
+      # puts "Opened: #{!@csvFile.nil?} - #{@@csvFileName}"
       # --
       @csvFileFull = CSV.open(@@fullFileName,'w')
       @csvFileFull << @@fullHeader
       puts "Opened: #{!@csvFileFull.nil?} - #{@@fullFileName} "
       # --
-      @csvFileTech = CSV.open(@@tallFileName,'w')
-      @csvFileTech << @@tallHeader
-      puts "Opened: #{!@csvFileTech.nil?} - #{@@tallFileName}"
+      # @csvFileTech = CSV.open(@@techFileName,'w')
+      # @csvFileTech << @@techHeader
+      # puts "Opened: #{!@csvFileTech.nil?} - #{@@techFileName}"
       # --
       @recNum      = 0
       @dsCount     = 0
@@ -106,11 +105,12 @@ module DataSources
       raise ArgumentError.new("ERROR in Workbok processing: '#{twb}' must be a Workbook (class) or the name of a Workbook (String), is a #{twb.class} \n ") unless @twb.is_a? Twb::Workbook
       # --
       dss = @twb.datasources
+      @fieldCnt = 0
       dss.each do |ds|
         @dsname = ds.uiname
         # puts "\n    -- #{ds.uiname} "
         # tables = Set.new
-        fields = {}
+        # fields = {}
         @dsCount += 1
         fclasses = Set.new
         ds.localFields.each do |field|
@@ -150,17 +150,17 @@ module DataSources
           # emitTech( ds,     field.uiname, 'Mapped', field.properties)
           # recordTech( field, 'MappedA')
         end
-        emitFields(fields)
+        # emitFields(fields)
       end
     end # def processTwb twb
 
     def recordFieldFull field, source
       # print field.properties.nil? ? '-' : ":#{field.properties.length}"
       # puts field.properties
-      @recNum+=1
+      @fieldCnt+=1
       field.properties.each do |name,value|
         # print name 
-        @csvFileFull << [  @recNum,
+        @csvFileFull << [  @recNum+=1,
                            @twb.name,
                            @twb.dir,
                            @dsname,
@@ -175,68 +175,68 @@ module DataSources
       end
     end
 
-    def recordField fields, field, props
-      # puts "%-65s  :: %s " % [fieldName,props]
-      return if field.uiname.nil?
-      if fields.has_key? field.uiname
-        fields[field.uiname].merge! field.properties
-      else
-        fields[field.uiname] = field.properties
-      end
-    end
-
-    def emitFields fields
-      fields.each do |fieldName,props|
-        # puts "FIELD::  %-40s  :: %s" % [fieldName,props.inspect]
-        # class = props[]
-        csvRec = [ @recNum+=1,
-                   @twb.name,
-                   @twb.dir,
-                   @dsname,
-                   fieldName,
-                   props[:type],
-                   props['hidden'],
-                   props[:columnField],
-                   props[:calculatedField],
-                   props[:dbField],
-                   props[:mappedField],
-                   props[:metadataField]
-                 ]
-        @csvFile << csvRec
-        @csvRecords.add csvRec
-      end
-    end
-
-    def recordTech field, type
-      @recNum+=1
-      field.properties.each do |name,value|
-        @csvFileTech << [ @recNum,
-                          @twb.name,
-                          @twb.dir,
-                          @dsname,
-                          field.uiname,
-                          type,
-                          name,
-                          value
-                        ]
-      end
-    end
-
-    def emitTech dataSource, fieldName, type, properties
-      # puts "XX #{dataSource.uiname} :: #{fieldName} #{}  :: #{type} :: #{properties}"
-      @recNum+=1
-      properties.each do |name,value|
-        @csvFileTech << [ @recNum,
-                          @twb.name,
-                          @twb.dir,
-                          dataSource.uiname,
-                          fieldName,
-                          type,
-                          name,
-                          value
-                        ]
-      end
-    end
+    # def recordField fields, field, props
+    #   # puts "%-65s  :: %s " % [fieldName,props]
+    #   return if field.uiname.nil?
+    #   if fields.has_key? field.uiname
+    #     fields[field.uiname].merge! field.properties
+    #   else
+    #     fields[field.uiname] = field.properties
+    #   end
+    # end
+
+    # def emitFields fields
+    #   fields.each do |fieldName,props|
+    #     # puts "FIELD::  %-40s  :: %s" % [fieldName,props.inspect]
+    #     # class = props[]
+    #     csvRec = [ @recNum+=1,
+    #                @twb.name,
+    #                @twb.dir,
+    #                @dsname,
+    #                fieldName,
+    #                props[:type],
+    #                props['hidden'],
+    #                props[:columnField],
+    #                props[:calculatedField],
+    #                props[:dbField],
+    #                props[:mappedField],
+    #                props[:metadataField]
+    #              ]
+    #     @csvFile << csvRec
+    #     @csvRecords.add csvRec
+    #   end
+    # end
+
+    # def recordTech field, type
+    #   @recNum+=1
+    #   field.properties.each do |name,value|
+    #     @csvFileTech << [ @recNum,
+    #                       @twb.name,
+    #                       @twb.dir,
+    #                       @dsname,
+    #                       field.uiname,
+    #                       type,
+    #                       name,
+    #                       value
+    #                     ]
+    #   end
+    # end
+
+    # def emitTech dataSource, fieldName, type, properties
+    #   # puts "XX #{dataSource.uiname} :: #{fieldName} #{}  :: #{type} :: #{properties}"
+    #   @recNum+=1
+    #   properties.each do |name,value|
+    #     @csvFileTech << [ @recNum,
+    #                       @twb.name,
+    #                       @twb.dir,
+    #                       dataSource.uiname,
+    #                       fieldName,
+    #                       type,
+    #                       name,
+    #                       value
+    #                     ]
+    #   end
+    # end
 
     def cleanup
       @csvFile.close unless @csvFile.nil?

data/lib/twb/analysis/DataSources/googlesheetdatasourcesanalyzer.rb

@@ -0,0 +1,79 @@
+#  sheetfieldsanalyzer.rb Copyright (C) 2018  Chris Gerrard
+#
+#  This program is free software: you can redistribute it and/or modify
+#  it under the terms of the GNU General Public License as published by
+#  the Free Software Foundation, either version 3 of the License, or
+#  (at your option) any later version.  
+#
+#  This program is distributed in the hope that it will be useful,
+#  but WITHOUT ANY WARRANTY; without even the implied warranty of
+#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#  GNU General Public License for more details.
+#
+#  You should have received a copy of the GNU General Public License
+#  along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+require 'twb'
+require 'csv'
+require 'set'
+
+module Twb
+module Analysis
+
+  class GoogleSheetDataSourcesAnalyzer
+
+    include TabTool
+
+    attr_accessor :localEmit
+    attr_reader   :twbname, :twbcount
+
+    def initialize
+      @twbcount   = 0
+      @sheetCnt   = 0
+      @filterCnt  = 0
+      @funcdoc    = {:class=>self.class, :blurb=>'Analyzing Google Sheet Data Sources from Tableau Workbooks.', :description=>nil,}
+      docFileName = docFile('TWBGoogleSheetDataSources.csv')
+      @csv        = CSV.open(docFileName, 'w')
+      @csv       << ["Workbook",'Data Source','Connection','File Name','Type','Table Name','Field']
+      @docfiles = [{:name=>docFileName,:desc=>"CSV File containing the data relating Google Sheet-based Data Sources."}]
+    end
+
+    def processTWB twb
+      if Twb::Workbook != twb.class
+        @twb = Twb::Workbook.new twb
+      else
+        @twb = twb
+      end
+      @twbname = @twb.name
+      emit  "Workbook:: #{@twbname}"
+      @twbcount += 1
+      dss        = twb.datasources
+      dss.each do |ds|
+          emit ds.uiname
+          conns = ds.node.xpath(".//connection[@class='google-sheets']")
+          if conns.length > 0
+              @relation = ds.node.at_xpath('./connection/relation')
+              @relName  = @relation.attribute('name').text
+              @relType  = @relation.attribute('type').text
+              @fileName = ds.node.at_xpath('.//named-connection/connection').attribute('filename')
+              emit "FILENAME: #{@fileName}"
+              tables = ds.node.xpath(".//relation[@type='table']")
+              # emit "# Tables: #{tables.length}"
+              tables.each do |table|
+                tableName = table.attribute('name')
+                columns   = table.xpath('.//column')
+                columns.each do |column|
+                  emit    [@twbname,ds.uiname,@relName,@fileName,@relType,tableName,column.attribute('name')].to_csv
+                  @csv << [@twbname,ds.uiname,@relName,@fileName,@relType,tableName,column.attribute('name')]
+                  emit ''
+                end
+              end
+          end
+          emit " "
+      end
+    end
+
+  end #class SheetFieldsAnalyzer
+
+end # module Analysis
+end # module Twb

data/lib/twb/analysis/DocumentedFieldsMarkdownEmitter.rb

@@ -28,7 +28,7 @@ module Analysis
 
     def processTwb twbName
       @twb = Twb::Workbook.new twbName
-      @docFileName = @twb.name + '.DocumentedFields.md'
+      @docFileName = '.ttdoc' + @twb.name + '.DocumentedFields.md'
       @docFile     = File.open(@docFileName,'w')
       @docFile.puts "## #{@twb.name}"
       dss = @twb.datasources