twb 2.2.1 → 3.7.2

data/lib/twb/analysis/Sheets/sheetfieldsanalyzer.rb

@@ -0,0 +1,82 @@
+#  sheetfieldsanalyzer.rb Copyright (C) 2018  Chris Gerrard
+#
+#  This program is free software: you can redistribute it and/or modify
+#  it under the terms of the GNU General Public License as published by
+#  the Free Software Foundation, either version 3 of the License, or
+#  (at your option) any later version.  
+#
+#  This program is distributed in the hope that it will be useful,
+#  but WITHOUT ANY WARRANTY; without even the implied warranty of
+#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#  GNU General Public License for more details.
+#
+#  You should have received a copy of the GNU General Public License
+#  along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+require 'twb'
+require 'csv'
+require 'set'
+
+module Twb
+module Analysis
+
+  class SheetFieldsAnalyzer
+
+    include TabTool
+
+    attr_accessor :localEmit
+
+    def initialize
+      @twbCnt     = 0
+      @sheetCnt   = 0
+      @filterCnt  = 0
+      @funcdoc    = {:class=>self.class, :blurb=>'Analyzing Sheet Fields from Tableau Workbooks.', :description=>nil,}
+      docFileName = docFile('TWBWorksheetFields.csv')
+      @sheetFieldsCSV  = CSV.open(docFileName,'w')
+      @sheetFieldsCSV  << ['Workbook','Worksheet','Data Source','Data Source (tech)','Field','Field (tech)']
+      @docfiles = [{:name=>docFileName,:desc=>"CSV File containing the data relating Workbooks,Worksheets, and the sheets' Data Sources and Fields"}]
+    end
+
+    def processTWB twb
+       @twb = twb
+       emit "   -- #{@twb.name}"
+       @twbCnt += 1
+       @twbDomainsLoaded = false
+       parseSheets
+    end
+
+
+    def parseSheets
+      @worksheets  = @twb.worksheets
+      @worksheets.each do |sheet|
+        emit "SHEET: #{sheet.name}"
+        showFields sheet unless sheet.datasourceFields.nil?
+      end
+    end
+
+    def showFields sheet
+      fields = sheet.datasourceFields
+      emit " #FIELDS: #{fields.length}"
+      if fields.nil?
+        @sheetFieldsCSV  << [@twb.name, sheet.name, nil, nil, nil, nil]
+      end
+      fields.each do |dsName, dsfields|
+        ds = @twb.datasource dsName
+        emit "      ds: #{dsName}"
+        emit "        - #{ds.uiname}"
+        emit "        : #{ds.class}"
+        dsfields.each do |sheetField|
+           emit "       f: #{sheetField}"
+           emit "       c: #{sheetField.class}"
+           fuiName = ds.fieldUIName sheetField #Fields[sheetField]
+           @sheetFieldsCSV  << [@twb.name, sheet.name, ds.uiname, dsName, sheetField.uiname, sheetField.name]
+           # emit true, "        : #{dsFields[field].class}"
+        end
+      end
+    end
+
+
+  end #class SheetFieldsAnalyzer
+
+end # module Analysis
+end # module Twb

data/lib/twb/analysis/Sheets/sheetfiltersanalyzer.rb

@@ -0,0 +1,214 @@
+#  WorksheetFields.rb Copyright (C) 2017, 2018  Chris Gerrard
+#
+#  This program is free software: you can redistribute it and/or modify
+#  it under the terms of the GNU General Public License as published by
+#  the Free Software Foundation, either version 3 of the License, or
+#  (at your option) any later version.  
+#
+#  This program is distributed in the hope that it will be useful,
+#  but WITHOUT ANY WARRANTY; without even the implied warranty of
+#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#  GNU General Public License for more details.
+#
+#  You should have received a copy of the GNU General Public License
+#  along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+require 'twb'
+require 'csv'
+require 'set'
+
+module Twb
+module Analysis
+
+  class SheetFiltersAnalyzer
+
+    include TabTool
+
+    attr_accessor :localEmit
+
+    def initialize
+      @twbCnt    = 0
+      @sheetCnt  = 0
+      @filterCnt = 0
+      # @ttlogfile       = File.new('./ttdoc/SheetFiltersAnalyzer.ttlog','w')
+      $sheetFieldsCSV  = CSV.open(docFile('TWBWorksheetFilters.csv')  ,'w')
+      $sheetFieldsCSV  << ['Workbook','Worksheet','Filter Type','Data Source','Field','Value','Alias', 'Alias?']
+    end
+
+    def processTWB twb
+       @twb = twb
+       emit "   -- #{@twb.name}"
+       @twbCnt += 1
+       @twbDomainsLoaded = false
+       parseFilters
+    end
+
+
+    def parseFilters
+      @worksheets  = @twb.worksheets
+      @worksheets.each do |sheet|
+        emit "\n\nSHEET: #{sheet.name}"
+        @sheetCnt += 1
+        filters = sheet.node.xpath('.//filter[@column]')
+        filters.each do |filter|
+          emit "\nFILTER:\n#{filter}\n====="
+          fieldCode  = filter.attribute('column').text
+          codedField = Twb::CodedField.new(fieldCode) 
+          fieldTech  = codedField.name
+          srcTech    = codedField.dataSource
+          dataSource = @twb.datasource(srcTech)
+          @dsAliases = dataSource.aliases
+          field      = dataSource.fieldUIName fieldTech
+          emit "\n::  FIELD  :: #{field}   --  #{fieldTech}  --  #{codedField.rawCode}"
+          filterValues = collectFilterValues sheet, dataSource, field, filter
+        end
+      end
+    end
+
+    def collectFilterValues sheet, dataSource, field, node
+      emit "collectFilterValues"
+      emit "              sheet: #{sheet.name}"
+      emit "                 ds: #{dataSource.uiname}"
+      emit "              field: #{field}"
+      emit "               node: #{node}"
+      emit " "
+      firstChild = node.at_xpath('./groupfilter')
+      results = {}
+      unless firstChild.nil?
+          function   = firstChild.attribute('function').text
+          emit  "function: #{function}"
+          emit  node.to_s
+          #-- single element filter
+          if 'member'.eql? function
+            emit  "HANDLING SINGLE MEMBER FILTER"
+            value  = firstChild.attribute('member').text.gsub(/^"|"$/,'') 
+            alia   = dataSource.deAlias(field,value)
+            emit  "value :%-25s =>  alias: %-s" % [value, alia]
+            $sheetFieldsCSV << [ @twb.name, sheet.name, 'single', dataSource.uiname, field, value, alia  ]
+            return  {value => alia}
+          end
+          #-- otherwise filter contains multiple elements
+          #-- handle individual member elements
+          elements = firstChild.xpath('.//groupfilter')
+          elements.each do |element|
+            function = element.attribute('function').text
+            emit "firstCHild\nfunction: #{function}\n    node:\n#{element}"
+            if 'member'.eql? function
+              name  =  element.attribute('member').text.gsub(/^"|"$/,'')
+              emit "%%%% NAME:: #{name}"
+              alia  =  dataSource.fieldAlias(field,name) # $TableNameAliases[name]
+              results[name] = alia
+            end
+            if 'range'.eql? function
+              t = filtersFromRangeNode(dataSource, field, element)
+              t.each do |name,alia|
+                emit "RANGE ELEMENT  Name: %-20s   ALIAS: %-s " % [name, alia]
+                results[name] = dataSource.fieldAlias(field,name)
+              end
+            end
+          end
+          results.each do |name,alia|
+            $sheetFieldsCSV << [ @twb.name, sheet.name, 'single', dataSource.uiname, field, name, alia, !name.eql?(alia) ]
+          end
+      end
+      return results
+    end
+
+    def filtersFromRangeNode dataSource, field, node
+      unless @twbDomainsLoaded
+        loadDomains 
+      end
+      emit  "filtersFromRangeNode"
+      emit  "    from: #{node.attribute('from')}"
+      emit  "    to. : #{node.attribute('to')}"
+      from = node.attribute('from').text.gsub(/^"|"$/,'')
+      to   = node.attribute(  'to').text.gsub(/^"|"$/,'')
+      emit  "    from: #{from}"
+      emit  "    to. : #{to}"
+      filtersInRange dataSource, field, from, to
+    end
+
+    def filtersInRange dataSource, field, from, to
+      emit  "filtersInRange"
+      results   = {}
+      dsFields  = @twbFielddomains[dataSource.uiname]
+      emit  "dsFields    : #{dsFields}"
+      if dsFields.nil? || dsFields.empty?
+          emit  "#### ALERT - FIELD DOMAIN VALUES FOR DATASOURCE #{dataSource.uiname} NOT LOADED ####"
+          emit  @twbFieldDomains
+          emit  "==========="
+          emit  dsFields
+          emit  "==========="
+      else
+          fieldVals = dsFields[field].to_a
+          if dataSource.fieldHasAliases field
+            #-- resolve aliases
+            dbValues = SortedSet.new
+            aliases  = dataSource.fieldAliases field
+            fieldVals.each do |fv|
+              fvAliased = aliases.has_value? fv
+              if fvAliased
+                dbValues <<  aliases.key(fv)
+              else 
+                dbValues <<  fv
+              end
+            end
+            $fieldVals = dbValues.to_a
+          else
+            #-- use domain values as returned
+            $fieldVals = dsFields[field].to_a
+          end
+          # domainVals = dsFields[field].to_a
+          # #-- need to dealias (unalias?) the field domain values
+          # dbVals = SortedSet.new
+          # domainVals.each do |dv|
+          emit  "field values: #{$fieldVals}"
+          unless $fieldVals.nil? || $fieldVals.empty?
+              values = $fieldVals
+              f_i    = values.index from
+              t_i    = values.index to
+              emit  "      from: #{f_i} :: '#{from}'"
+              emit  "        to: #{t_i} :: '#{to}'"
+              badRange = f_i.nil? || t_i.nil?
+              emit "badRange: #{badRange}"
+              if badRange
+                emit  "BAD RANGE"
+              else
+                tnames  = values[f_i..t_i]
+                tnames.each do |tname|
+                  results[tname] = dataSource.deAlias(field,tname) # $TableNameAliases[tname]
+                end
+                emit  " filtersInRange  f:%-35s  t:%-s " % ["'#{from}:#{f_i}'", "'#{to}:#{t_i}'"]
+                emit  "         names:  #{results.keys}"
+                emit  "       aliases:  #{results.values}"
+              end
+          end
+      end
+      return results
+    end
+
+    def loadDomains
+      emit  "def loadDomains\n=="
+      loader            = Twb::Util::FieldDomainLoader.new
+      @twbFielddomains  = loader.loadWorkbook @twb
+      emit  "#{@twbFielddomains}\n=="
+      @twbDomainsLoaded = true
+    end
+
+      # def emit(local=@localEmit, stuff)
+      #     if stuff.is_a? String then
+      #       lines = stuff.split(/\n/)
+      #       lines.each do |line|
+      #         @ttlogfile.puts "#{$emitPrefix}#{line}"
+      #                    puts "#{$emitPrefix}#{line}" if local
+      #       end
+      #     else
+      #         @ttlogfile.puts "#{$emitPrefix}#{stuff}"
+      #                    puts "#{$emitPrefix}#{stuff}" if local
+      #     end
+      # end
+
+end # class 
+
+end # module Twb
+end # module Analysis

data/lib/twb/calculatedfield.rb

@@ -14,15 +14,16 @@
 #  along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
 require 'nokogiri'
+require 'digest/md5'
 
 module Twb
 
-  class CalculatedField
+  class CalculatedField < TabClass
 
     attr_reader :dataSource
     attr_reader :node,     :properties
-    attr_reader :caption,  :name, :uiname  
-    attr_reader :datatype, :role, :type  
+    attr_reader :caption,  :name, :uiname
+    attr_reader :datatype, :role, :propType  
     attr_reader :calculation, :calcFields
     attr_reader :hidden
 
@@ -36,7 +37,7 @@ module Twb
         # -- 
         @datatype    = @node.attribute('datatype').text
         @role        = @node.attribute('role').text
-        @type        = @node.attribute('type').text
+        @propType    = @node.attribute('type').text   # n.b. 'type' is used as a proxy for class
         # -- 
         @calculation = Twb::FieldCalculation.new(self, datasource)
         # -- 
@@ -47,7 +48,7 @@ module Twb
       @properties ||= loadProperties
     end
 
-    def calcLines
+    def calcFields
         @calculation.calcFields
     end
 
@@ -69,9 +70,23 @@ module Twb
         @properties[name] = attr.value
       end
       @properties[:uiname] = @uiname
+      @properties[:uuid]   = uuid
       return @properties
     end
 
+    def id
+        @id ||= @id = "#{@dataSource.uiname}::#{@uiname}"
+    end
+
+    # def uuid
+    #     @uuid ||= loadUUID
+    # end
+
+    # def loadUUID
+    #     dsn = @dataSource.nil? ? 'NO DATASOURCE' : dataSource.uuid 
+    #     @uuid = Digest::MD5.hexdigest("#{dsn}::@name").hash
+    # end
+
     def to_s
         "%s(%s) => %s" % [uiname, name, @calculation.formulaFlat]
     end

data/lib/twb/codedfield.rb

@@ -0,0 +1,87 @@
+#  Copyright (C) 2014, 2015, 2017  Chris Gerrard
+#
+#  This program is free software: you can redistribute it and/or modify
+#  it under the terms of the GNU General Public License as published by
+#  the Free Software Foundation, either version 3 of the License, or
+#  (at your option) any later version.
+#
+#  This program is distributed in the hope that it will be useful,
+#  but WITHOUT ANY WARRANTY; without even the implied warranty of
+#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#  GNU General Public License for more details.
+#
+#  You should have received a copy of the GNU General Public License
+#  along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+require 'nokogiri'
+require 'digest/md5'
+require 'csv'
+
+module Twb
+
+  class CodedField
+
+    include Comparable
+
+    attr_reader :dataSource
+    attr_reader :name, :techCode, :rawCode 
+
+    def initialize code
+     #puts "\n\nCodedField ::  #{code}"
+      @rawCode = code
+      trimmed  = code.gsub(/^"|"$/,'').gsub(/^\[|\]$/,'')
+      parts    = trimmed.split('].[')
+     #puts "Field: #{code}  parts: #{parts.length} - #{parts.inspect}"
+     #puts "   p1: #{parts[0]}"
+     #puts "   p2: #{parts[1]}"
+      if parts.length == 1
+       #puts '==1'
+        @name = parts[0]
+        @techCode = "[#{@name}]"
+      else # parts.length <> 1
+       #puts '<>1'
+       #puts "p[0]: #{parts[0]}"
+       #puts "p[1]: #{parts[1]}"
+        @dataSource = parts[0]
+        fldName     = parts[1]
+        if fldName.start_with?(':') && fldName.count(':') == 1
+          @name = fldName
+        else
+          parseField fldName
+        end
+        @techCode   = "[#{@dataSource}].[#{@name}]"
+      end
+    end # initialize
+
+    def parseField str
+      # puts "parseField: #{str}"
+      parts = str.split(':')
+      # puts "parseField: #{str}"
+      # puts "    parts : #{parts}"
+      # puts "    partsl: #{parts.length}"
+      # puts "      p[0]: #{parts[0]}"
+      # puts "      p[1]: #{parts[1]}"
+      case parts.length
+      when 1
+        @name = parts[0]
+      else
+        @name = parts[1]
+      end
+    end
+
+    def id
+      @id ||= @id = "#{@dataSourceName}::#{@uiname}"
+    end
+
+    def <=>(other)
+      # myName    = @uiname.nil? ? '' : @uiname
+      # otherName = other.uiName.nil? ? "" : other.uiName
+      # ##puts "#{@uiname} / #{myName} <=>  #{otherName} / #{other.uiName}"
+      # ##puts "#{@uiname.nil?} // #{other.uiName.nil?}"
+      # myName <=> otherName
+      @fqName <=> other.techCode
+    end
+
+  end # class CalculationField
+
+end  # module Twb

data/lib/twb/columnfield.rb

@@ -17,7 +17,7 @@ require 'nokogiri'
 
 module Twb
 
-  class ColumnField
+  class ColumnField < TabClass
 
     include Comparable
 
@@ -54,10 +54,10 @@ module Twb
     attr_reader :alias,      :semanticRole,  :aggregation
     attr_reader :autoColumn, :hidden,        :datatypeCustomized
     attr_reader :calcField
+    attr_reader :aliases
 
 
     def initialize(fieldNode, datasource=nil)
-      
       @datasource         = datasource
       @node               = fieldNode
       @name               = load 'name'
@@ -79,6 +79,10 @@ module Twb
       @calcField          = loadCalcField
     end
 
+    def id
+        @id ||= @id = @name.hash
+    end
+
     def load nodeName
       attr = @node.attribute(nodeName)
       val  = attr.nil? ? nil : attr.text.strip.gsub(/^\[|\]$/,'')
@@ -135,6 +139,21 @@ module Twb
       @commentLines.empty?
     end
 
+    def aliases
+      @aliases ||= loadAliases
+    end
+
+    def loadAliases
+      @aliases = {}
+      aliasNodes = @node.xpath('.//alias')
+      aliasNodes.each do |node|
+        key   = node.attribute('key').text.gsub(/^"|"$/,'')
+        value = node.attribute('value').text.gsub(/^"|"$/,'')
+        @aliases[key] = value
+      end
+      return @aliases
+    end
+
     def properties
       @properties ||= loadProperties
     end