twb 3.9.7 → 4.3.1

data/lib/twb/datasource.rb

@@ -42,8 +42,11 @@ module Twb
 
     attr_reader :workbook
     attr_reader :name,          :caption,         :uiname
+    attr_reader :attributes
     attr_reader :dsclass,       :isExtract
-    attr_reader :connection,    :connections,     :connHash,        :uuid
+    attr_reader :connection,    :connections,     :connHash,   :connAttributes       
+    attr_reader :isPublished
+    attr_reader :uuid
     attr_reader :tables,        :joinPairs      
     attr_reader :localFields,   :localFieldNames, :localField, :hasField 
     attr_reader :columnFields 
@@ -73,6 +76,24 @@ module Twb
       return self
     end
 
+    def attributes
+      @attributes ||= @attributes = loadAttributes(@node) 
+    end
+
+    def connAttributes
+      @connAttributes ||= @connAttributes = loadAttributes(@connection)
+    end
+
+    def loadAttributes node
+      attributes = {}
+      unless node.nil?
+        node.attributes.each do |k,v|
+          attributes[k] = v.text
+        end
+      end
+      return attributes
+    end
+
     def id
         @id ||= @id = @name
     end
@@ -145,6 +166,14 @@ module Twb
       @isExtract ||= @isExtract = !@node.at_xpath('./extract').nil?
     end
 
+    def isPublished
+      @isPublished ||= loadIsPublished
+    end
+
+    def loadIsPublished
+      @isPublished = !node.at_xpath('./repository-location').nil?
+    end
+
     def loadTables connection
       @tables = {}
       nodes = connection.xpath(".//relation[@type='table']")
@@ -269,8 +298,10 @@ module Twb
     end
 
     def fieldAlias fieldName, value
-      return value unless fieldHasAliases(fieldName)
-      fldAlias = aliases[fieldName][value]
+      emit "fieldAlias:  #{fieldName.class}  -> #{value.class}" 
+      loadAliases  if @aliases.nil?
+      return value if @aliases.nil? || @aliases[fieldName].nil? # unless fieldHasAliases(fieldName)
+      fldAlias = @aliases[fieldName][value]
       fldAlias.nil? ? value : fldAlias
     end
 
@@ -336,7 +367,7 @@ module Twb
 
     def fieldUIName fieldName
       loadFieldUINames if @fieldUINames.nil?
-      @fieldUINames[fieldName]
+      @fieldUINames[fieldName].nil? ? fieldName : @fieldUINames[fieldName]
     end
 
     def fieldUINames

data/lib/twb/fieldcalculation.rb

@@ -38,7 +38,7 @@ module Twb
     attr_reader :fields,   :remoteFields,  :calcFields
     attr_reader :comments, :uuid
 
-    attr_accessor :ttlogfile
+    # attr_accessor :ttlogfile
 
     @@tableCalcs = [ 'FIRST',        'INDEX', 'LAST', 'SIZE',
                      'LOOKUP',       'PREVIOUS_VALUE',
@@ -53,16 +53,15 @@ module Twb
                      'WINDOW_VAR',   'WINDOW_VARP'
                    ]
 
-    @ttlogfile = 
-
     def initialize(calcField, datasource=nil)
       raise ArgumentError.new("FieldCalculation must be initialized with a CalculatedField, has been provided with a #{calcField.class}") if calcField.class != Twb::CalculatedField
+      init
       # initLogger
       @field        = calcField
       calcNode      = calcField.node
       @istableCalc  = false
       # @localEmit    = false
-      # puts "FieldCalculation calcNode.nil? :: #{calcNode.nil?} "
+      emit "FieldCalculation calcNode.nil? :: #{calcNode.nil?} "
       unless calcNode.nil?
         @node         = calcNode.at_xpath('./calculation')
         @fieldNode    = calcField.node
@@ -94,7 +93,7 @@ module Twb
           @has_formula    = @node.has_attribute?('formula')
           if @has_formula
             @formula      = @node.attribute('formula').text.gsub(/\r\n/,"\n")
-            # puts "\n-- init: #{@formula}"
+            emit "\n-- init: #{@formula}"
             @formulaUC    = @formula.upcase
             @formulaLines = formula.split(/\n|\r\n/)
             @formulaFlat  = flattenFormula(@formulaLines)
@@ -108,12 +107,6 @@ module Twb
       end
     end
 
-    # def initLogger(logfile=nil)
-    #   logfilename   = docFile(logfile.nil? ? @ttlogfile : logfile)
-    #   @logger       = Logger.new(logfilename)
-    #   @logger.level = Logger::DEBUG
-    # end
-
     def id
         @id ||= @id = @formulaFlat.hash + calcField.hash
     end
@@ -122,10 +115,6 @@ module Twb
       @uuid ||= @uuid = Digest::MD5.hexdigest(@formula)
     end
 
-    # def assessTableCalc formula
-    #   @@tableCalcs.any? { |tc| string.include?(tc) }
-    # end
-
     def attribText(node, attribute)
       node.attribute(attribute).nil? ? nil : node.attribute(attribute).text
     end

data/lib/twb/quickfilter.rb

@@ -23,7 +23,7 @@ module Twb
     include TabTool
 
     attr_reader :field,     :name,      :uiname
-    attr_reader :type
+    attr_reader :type,      :kind
     attr_reader :dataSource
     attr_reader :node,      :values
     attr_reader :inexclude, :inexMode,  :includeNull
@@ -32,8 +32,9 @@ module Twb
       init
       emit "\nFILTER:\n#{node}\n====="
       @node         = node
-      filterClass   = node['class']
+      filterClass   = @node['class']
       @type         = filterClass.gsub('-',' ').capitalize
+      @kind         = @node['kind']
       fieldCode     = node['column']
       codedField    = Twb::CodedField.new(fieldCode) 
       fieldTech     = codedField.name
@@ -83,7 +84,7 @@ module Twb
         when 'quantitative'  then resolveQuantitative
         when 'categorical'   then resolveCategoricalValues
       end
-    end
+    end # def initialize
 
     def to_s
         "%s => %s" % [uiname, values]
@@ -93,7 +94,7 @@ module Twb
 
     def recordValue value, valias=nil
       # puts "recordValue--: #{value} :: @'#{valias}'"
-      valias = value if valias.nil?
+      # valias = value if valias.nil?
       @values <<  {:value => value, :alias => valias}
       # puts "recordValue--: done"
     end
@@ -194,9 +195,8 @@ module Twb
     # <filter class='categorical' column='[Sample - Superstore - English (Extract)].[none:Region:nk]'>
     #   <groupfilter from='&quot;East&quot;' function='range' level='[none:Region:nk]' to='&quot;West&quot;' user:ui-domain='relevant' user:ui-enumeration='inclusive' user:ui-marker='enumerate' />
     # </filter>
-    #--
     def resolveCategoricalValues
-      emit "resolveCategoricalValues"
+      emit "########################## resolveCategoricalValues"
       emit "@measureNames: #{@measureNames}"
       if @node.element_children.empty?
         # <filter class='categorical' column='[Sample - Superstore].[Top Customers by Profit (copy)]' />
@@ -337,8 +337,8 @@ module Twb
                 recordValue range, range
               end 
               t.each do |name,alia|
-                # emit "%%%% range Name: %-20s   ALIAS: %-s " % [name, alia]
-                recordValue name, @dataSource.fieldAlias(@uiname,name)
+                # emit true, "++++ range Name: %-20s   ALIAS: %-s " % [name, alia]
+                # recordValue name, @dataSource.fieldAlias(@uiname,name)
               end
             end
           end
@@ -346,6 +346,7 @@ module Twb
     end
 
     def filtersFromRangeNode node
+      emit "########################## filtersFromRangeNode"
       unless @twbDomainsLoaded
         loadDomains 
       end
@@ -368,7 +369,7 @@ module Twb
     end
 
     def filtersInRange from, to
-      emit  "filtersInRange"
+      emit  "########################## filtersInRange"
       # results   = {}
       dsFields  = @twbFielddomains[@dataSource.uiname]
       emit  "dsFields    : #{dsFields}"

data/lib/twb/tabtool.rb

@@ -16,12 +16,14 @@
 require 'logger'
 require 'nokogiri'
 require 'digest/md5'
+require 'yaml'
 
 module TabTool
 
   @@licensed = false
 
-  TTDOCDIR   = './ttdoc'
+  @@TTDOCDIR    = './ttdoc'
+  @configFile = './ttconfig.yml'
 
   attr_accessor :ttdocdir, :logger,   :loglevel, :logfilename
   attr_accessor :uuid,     :type,     :id,       :properties
@@ -42,9 +44,9 @@ module TabTool
   end
 
   def initDocDir
-    # return if TTDOCDIR.nil?
-    # return if ''.eql?($ttdocdir) && ''.eql?(TTDOCDIR)
-    @docDir = TTDOCDIR
+    # return if @@TTDOCDIR.nil?
+    # return if ''.eql?($ttdocdir) && ''.eql?(@@TTDOCDIR)
+    @docDir = @@TTDOCDIR
     return if Dir.exists?(@docDir)
     if File.exists? @docDir
       @docDir = ''
@@ -55,9 +57,12 @@ module TabTool
   end
 
   def initLogger
-    logFileName   = docFile("#{self.class.to_s.split('::').last}.ttlog")
-    @logger       = Logger.new(logFileName, 2, 1000*1024)
-    @logger.level = Logger::DEBUG
+    @logger = nil
+    if config(:log)
+      logFileName   = docFile("#{self.class.to_s.split('::').last}.ttlog")
+      @logger       = Logger.new(logFileName)
+      @logger.level = Logger::DEBUG
+    end
     return @logger
   end
 
@@ -77,6 +82,23 @@ module TabTool
     @funcdoc.nil? ? {:class=>'n/a', :blurb=>'generic TabTool blurb', :description=>'A useful Tableau Tool.'} : @funcdoc
   end
 
+  def hasConfig param
+    loadConfig if @configParams.nil?
+    @configParams[param]  ? true : false
+  end
+
+  def config param
+    hasConfig(param) ? @configParams[param] : nil
+  end
+
+  def loadConfig
+    @configParams = if File.exist?('./ttconfig.yml') 
+                      YAML.load( File.open('./ttconfig.yml').read )
+                    else
+                      {}
+                    end
+  end
+
   def docfiles
     @docfiles ||= @docfiles = []
   end
@@ -95,21 +117,28 @@ module TabTool
     return maxlen
   end
 
-  def docfilesdoc
+  def docfilesdoc(pad=' ',desc='For documentation and generated data see the following:')
     lines   = SortedSet.new
+    paddesc = "#{pad}#{desc}"
     unless @docfiles.nil? || @docfiles.empty?
       nameLen = docFileMaxNameLen
       docfiles.each do |dfi|
-        lines << "    - %-#{nameLen}s    %-s " % [ dfi[:name], dfi[:description] ]
+        lines << "#{pad}- %-#{nameLen}s    %-s " % [ dfi[:name], dfi[:description] ]
       end
     end
-    docLines = lines.empty? ? [] : [' ',' For documentation and generated data see the following:',' ']
+    docLines = lines.empty? || paddesc =~ /^[ ]+/ ? [] : [' ',"#{pad}#{desc}"]
     lines.each do |l|
       docLines << l
     end
     return docLines
   end
 
+  def docfilesdocto_s(pad=' ',desc='For documentation and generated data see the following:')
+    str = ''
+    docfilesdoc(pad,desc).each { |l| str += "#{l}\n"}
+    return str
+  end
+
   # def metrics
   #   {}
   # end
@@ -154,6 +183,5 @@ module TabTool
     # @logger.close unless @logger.nil? || @logger.closed?
   end
 
-
 end # module TabTool
 

data/lib/twb/workbook.rb

@@ -1,4 +1,3 @@
-
 #  Copyright (C) 2014, 2018  Chris Gerrard
 #
 #  This program is free software: you can redistribute it and/or modify
@@ -25,11 +24,14 @@ module Twb
   #
   class Workbook < TabClass
 
-    attr_reader :workbooknode 
-    attr_reader :name,            :dir         
+    attr_reader :workbooknode,    :node 
+    attr_reader :name,            :dir,               :type         
     attr_reader :modtime,         :version,           :build
+    #--
     attr_reader :datasources,     :datasource 
     attr_reader :datasourceNames, :datasourceUINames, :dataSourceNamesMap
+    attr_reader :orphanDataSources  # i.e. not referenced in any Worksheet
+    #--
     attr_reader :dashboards,      :storyboards,       :worksheets
     attr_reader :parameters,      :actions
     attr_reader :valid,           :ndoc
@@ -66,18 +68,21 @@ module Twb
       Zip::File.open(twbxWithDir) do |zip_file|
         twb   = zip_file.glob('*.twb').first
         @ndoc = Nokogiri::XML(twb.get_input_stream)
+        @type = :twbx
         processDoc
       end
     end
 
     def processTWB(twbFile)
-      @ndoc    = Nokogiri::XML(open(twbFile))
+      @ndoc = Nokogiri::XML(open(twbFile))
+      @type = :twb
       processDoc
     end
 
     def processDoc
-      @workbooknode  = @ndoc.at_xpath('//workbook')
-      @version = @ndoc.xpath('/workbook/@version').first.text
+      @node         = @ndoc.at_xpath('//workbook')
+      @workbooknode = @node
+      @version      = @node.nil? ? nil : @node["version"]
       loaddatasources
       loadWorksheets
       loadDashboards
@@ -204,6 +209,20 @@ module Twb
       @dashboards[name]
     end
 
+    def orphanDataSources
+      @orphanDataSources ||= identifyOrphandatasoUrceS
+    end
+
+    def identifyOrphandatasoUrceS
+      sheetDataSources = Set.new
+      @worksheets.values.each do |sheet|
+        sheet.datasources.each do |ds|
+            sheetDataSources << ds.uiname
+        end
+      end
+      @orphanDataSources = @datasourceUINames - sheetDataSources
+    end
+
     def storyboards
       @storyboards.values
     end
@@ -283,12 +302,15 @@ module Twb
     # Write the TWB to a file, with an optional name.
     # Can be used to write over the existing TWB (dangerous), or to a new file (preferred).
     def write(name=@name)
-      $f = File.open(name,'w')
-      if $f
-          $f.puts @ndoc
-          $f.close
+      case @type
+      when :twb 
+        writeTwb(name)
+      when :twbx 
+        writeTwbx(name)
+      else
+        emit  "Cannot write this Workbook - it has an invalid type: #{@type}"
+        raise "Cannot write this Workbook - it has an invalid type: #{@type}"
       end
-      return name
     end
 
     # Write the TWB to a file, appending the base name with the provided string.
@@ -298,7 +320,22 @@ module Twb
       write newName
     end
 
-    private 
+    private
+
+    def writeTwb(name=@name)
+      $f = File.open(name,'w')
+      if $f
+          $f.puts @ndoc
+          $f.close
+      end
+      return name
+    end
+
+    def writeTwbx(name=@name)
+        emit  "Writing the Workbook, need implementation"
+    end
+
+
 
     def loadParameters
       @parameters = {}

data/lib/twb/worksheet.rb

@@ -24,16 +24,20 @@ module Twb
 
     @@hasher = Digest::SHA256.new
 
+    @fieldEncodingsXPath = './table/panes/pane//encodings'
+
     attr_reader :node,   :name,      :datasourcenames, :datasources
     attr_reader :panesCount
-    attr_reader :fields, :rowFields, :colFields, :paneFields, :datasourceFields
+    attr_reader :fields, :rowFields, :colFields, :paneFields, :datasourceFields, :pageFields, :encodedFields
     attr_reader :filters
+    attr_reader :tooltip
     attr_reader :hidden, :visible
 
     def initialize sheetNode, twb
       @twb  = twb
       @node = sheetNode
-      @name = @node.attr('name')
+      @name = @node['name']
+      emit "########################## Worksheet initialize name: #{@name}"
       loadDataSourceNames
       loadFields
       return self
@@ -77,6 +81,10 @@ module Twb
       @filters ||= loadFilters
     end
 
+    def tooltip
+      @tooltip ||= loadTooltip
+    end
+
     def hidden
       @hidden ||= resolveHidden
     end
@@ -105,8 +113,39 @@ module Twb
       @fields[:rows]   = @rowFields
       @colFields     ||= loadRowColFields(:cols) # returns map of data source => (Set of) field names
       @fields[:cols]   = @colFields
+      loadFieldEncodings
+    end
+
+    def loadFieldEncodings
+      @encodedFields = Hash.new { |h,k| h[k] = [] }
+      enodes = node.xpath('.//table/panes/pane/encodings')
+      enodes.each do |enode|
+        enode.children.each do |child|
+          unless child['column'].nil?
+            @encodedFields[child.name] << CodedField.new(child['column'])
+          end
+        end
+      end
+      @encodedFields.each do |type, fields|
+        @fields[type] = fields
+      end
     end
 
+    # def loadFieldEncodings node
+    #   $encodedFields = Hash.new { |h,k| h[k] = [] }
+    #   enodes = node.xpath('.//table/panes/pane/encodings')
+    #   enodes.each do |enode|
+    #     enode.children.each do |child|
+    #       unless child['column'].nil?
+    #         $encodedFields[child.name] << CodedField.new(child['column'])
+    #       end
+    #     end
+    #   end
+    #   $encodedFields.each do |type, fields|
+    #     $fields[type] = fields
+    #   end
+    # end
+
     def addDSFields fields, usage
       fields.each do  
       end
@@ -119,12 +158,16 @@ module Twb
     def loadRowColFields(type)
       fields = []
       xpath = case type
-        when :rows then './/rows'
-        when :cols then './/cols'
+        when :rows then './table/rows'
+        when :cols then './table/cols'
       end
       return fields if xpath.nil?
       node = @node.at_xpath(xpath)
-      RowsColsSplitter.new(node).fields.each do |fcode|
+      emit "def loadRowColFields:: type: #{type} \t node: #{node.class} \t  path: #{node.nil? ? '<<nil>>' : node.path }"
+      emit "node      : #{node.inspect}"
+      nodeFields = RowsColsSplitter.new(node).fields
+      emit "nodeFields: #{nodeFields}"
+      nodeFields.each do |fcode|
         fields << CodedField.new(fcode)
       end
       return fields
@@ -139,18 +182,32 @@ module Twb
       panes = @node.xpath('.//pane')
     end
 
+    def pageFields
+      @pageFields ||= loadpageFields
+    end
+
+    def loadpageFields 
+      @pageFields = [] # { |h,k| h[k] = [] }
+      nodes = node.xpath('.//table/pages/column')
+      nodes.each do |node|
+        @pageFields << CodedField.new(node.text)
+      end
+      return @pageFields
+    end
+
     def datasourcenames
       @datasources.keys
     end
 
     def resolveHidden
-      windowNode = node.at_xpath("//windows/window[@name='#{@name}']")
+      windowNode = node.at_xpath("//windows/window[@name=\"#{@name}\"]")
       @hidden    = !windowNode.nil? && 'true' == windowNode['hidden']
     end
 
     private
 
     def loadFilters
+      emit ""########################## loadFilters"
       @filters    = []
       filterNodes = @node.xpath('./table/view//filter[@column]')
       filterNodes.each do |fnode|
@@ -159,6 +216,10 @@ module Twb
       end
       return @filters
     end
+
+    def loadTooltip
+      @tooltip = @node.xpath('./table/panes/pane/customized-tooltip')
+    end
   
   end # class Worksheet
 
@@ -201,7 +262,9 @@ module Twb
       return fields if node.nil?
       unless node.text.nil?
         codes = node.text.split(/[\])] [\/+*] [(]?\[/)
-        codes.each {|c| @fields << c.gsub(/^[(]*[\[]*|[)\]]*$/,'')}
+        codes.each do |c| 
+          @fields << c.gsub(/^[(]*[\[]*|[)\]]*$/,'')
+        end
       end
     end
   end # class RowsColsSplitter