twb 0.5.3 → 1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: d8e5620e8ff52252576b9ba62c2c274341859698
+  data.tar.gz: de116f608f42f388d3527c09f23a2786c7193918
+SHA512:
+  metadata.gz: 65e5bda53ed6f4b1dc53e6dfce49fd6a70483369a5738ddd8fe8000d8aa997a4a4f09d5aff7082f96e71d0358f0c8dacfb68c6d46223d0f0acec3ee28defb7eb
+  data.tar.gz: fc675deeba6612bbaa5eb8523494abd580d65e3bceff019525e4a6519f5a66dedcd970dcf6c95b5ce340d86d5fa61ca0dbc504a638784ec4d2eb8451f50dff61

data/lib/twb/action.rb

@@ -0,0 +1,121 @@
+#  Copyright (C) 2014, 2016  Chris Gerrard
+#
+#  This program is free software: you can redistribute it and/or modify
+#  it under the terms of the GNU General Public License as published by
+#  the Free Software Foundation, either version 3 of the License, or
+#  (at your option) any later version.
+#
+#  This program is distributed in the hope that it will be useful,
+#  but WITHOUT ANY WARRANTY; without even the implied warranty of
+#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#  GNU General Public License for more details.
+#
+#  You should have received a copy of the GNU General Public License
+#  along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+require 'nokogiri'
+require 'digest/md5'
+
+module Twb
+
+  class Action
+
+    # Notes
+    # Sources:
+    # <source>
+    #   type="datasource"
+    #     <source datasource="Sample - Superstore - English (Extract)" type="datasource">
+    #       <exclude-sheets  - use to exclude these from the Workbook's sheets
+    #                        - haven't seen an example of named sheet for datasoure
+    #                        - can only target a single worksheet, by observation
+    #   type="sheet"
+    #     Worksheet
+    #     <source type="sheet" worksheet="Sheet 3"/>
+    #                        - use the worksheet attribute
+    #                        - can only target a single worksheet, by observation
+    #     Dashboard
+    #     <source dashboard="Dashboard 1" type="sheet" worksheet="Sheet 5"/>
+    #                        - dashboard attribute is present
+    #                        - use the worksheet attribute, if available
+    #     <source dashboard="Dashboard 2" type="sheet"/>
+    #       NO <exclude-sheets - source is all of the dashboard's sheets
+ 
+    @@hasher = Digest::SHA256.new
+
+    attr_reader :workbook, :node 
+    attr_reader :name, :caption, :uiname
+    attr_reader :type
+    attr_reader :sourceDash, :sourceType, :sourceSheet
+    attr_reader :cmdTarget,  :cmdExclude, :cmdSpecFields
+    attr_reader :linkExpression
+
+    def initialize actionNode, workbookNode
+      @workbook  = workbookNode
+      @node      = actionNode
+      # --
+      @name      = @node.attr('name')
+      @caption   = @node.attr('caption')
+      @uiname    = @caption.nil? ? @name : @caption
+      # --
+      @type      = setType
+      # --
+      @cmdTarget     = cmdParam 'target'
+      @cmdExclude    = cmdParam 'exclude'
+      @cmdSpecFields = cmdParam 'special-fields'
+      # --
+      @linkExpression = linkExpr
+      process
+      return self
+    end
+
+    def process
+
+    end
+
+    def setType
+      linkNode = @node.at_xpath('./link')
+      return :link unless linkNode.nil?
+      command  = @node.at_xpath('./command/@command')
+      return :notset if command.nil?
+      type = case command.text
+        when 'tsc:brush'
+          :highlight
+        when 'tsc:tsl-filter'
+          :filter
+        else
+          command.text
+      end
+    end
+
+    def linkExpr
+      attr = @node.at_xpath('./link/@expression')
+      attr.nil? ? nil : attr.text
+    end
+
+    def cmdParam param
+      n = @node.at_xpath("./command/param[@name='#{param}']")
+      return nil if n.nil?
+      v = n.attribute('value')
+      v.nil? ? nil : v.text
+    end
+
+    def sourceDash
+      attr = @node.at_xpath('./source/@dashboard')
+      attr.nil? ? nil : attr.text
+    end
+
+    def sourceSheet
+      attr = @node.at_xpath('./source/@worksheet')
+      attr.nil? ? nil : attr.text
+    end
+
+    def sourceType
+      @node.at_xpath('./source/@type')
+    end
+
+    def typeConv rawType
+    end
+
+  end
+
+end

data/lib/twb/countNodes.rb

@@ -0,0 +1,98 @@
+require 'nokogiri'
+require 'csv'
+require 'twb'
+
+def init
+  system 'cls'
+  $pFile = File.open('sqlFromTwbDc.txt','w')
+
+  sqiCSV = 'TWBFieldsByType.csv'
+  $fieldsCSV = CSV.open(sqiCSV, 'w')
+  $fieldsCSV    << [
+                    'TWB', 
+                    'Data Connection - UI', 
+                    'Field Name', 
+                    'Field Type', 
+                   ]
+  $path    = if ARGV.empty? then '**/*.twb' else ARGV[0] end
+  emit " "
+  emit " Generating SQL Create Table code for Tableau Workbook Data Connections (TWDCs)."
+  emit " Each TWDC will have it's own *.sql file containing the SQL code.\n "
+  emit " Looking for Workbooks matching: '#{$path}'  - from: #{ARGV[0]}\n\n "
+end
+
+def method_name
+    datasources     = {}
+    dataSourcesNode  = doc.at_xpath('//workbook/datasources')
+    dataSourcesNodes = doc.xpath('//workbook/datasources/datasource').to_a
+    puts "  dsn: #{dataSourcesNodes}"
+    datasourceNodes.each do |node|
+        datasource = Twb::DataSource.new(node)
+        datasources[datasource.name] = datasource
+    end
+end
+  
+$localEmit = true
+def emit stuff
+    $pFile.puts stuff
+           puts stuff if $localEmit
+end
+
+$paths = { 
+          'Relation Columns' => './connection/relation/columns/column',
+          'Metadata Records' => './connection/metadata-records/metadata-record',
+          'Columns'          => './column',
+         }
+
+def proc file
+  emit "\n == #{file}"
+end
+
+def process file
+  emit "\n == #{file}"
+  doc = Nokogiri::XML(open(file))
+  dataSourcesNodes = doc.xpath('//workbook/datasources/datasource')
+  dataSourcesNodes.each do |ds|
+    emit "\n   dc: #{ds.attribute('caption')}"
+    emit   "   dn: #{ds.attribute('name')}\n   ---"
+    typeCounts = {}
+    $paths.each do |type, path|
+      nodes = ds.xpath(path).to_a
+#      emit "     : %3i  %-17s  %-s" % [nodes.size, type, path]
+      typeCnt = nodes.size
+      nodes.each do |n|
+        $fieldsCSV << [file,ds.attribute('name'),n.attribute('name'),type]
+      end
+      if type == 'Columns' then 
+        calcCnt = 0
+        nodes.each do |n|
+          calc = n.xpath('./calculation')
+#          emit "    c:   #{calc.size} "
+          calcCnt += calc.size
+        end
+#        emit '   ---'
+#        emit "   c#: %3i " % [calcCnt]
+#        emit "  !c#: %3i " % [nodes.size - calcCnt]
+        typeCounts['Columns']            = nodes.size 
+        typeCounts['Columns - calc']     = calcCnt 
+        typeCounts['Columns - not calc'] = nodes.size - calcCnt 
+      else
+        typeCounts[type] = typeCnt 
+      end
+    end
+#    emit '   ---'
+    typeCounts.each do |t,c|
+      emit "   tc: %3i  %-s" % [c, t]
+    end
+  end
+end
+
+init
+
+$paths.each do |path|
+	emit "  p: #{path}"
+end
+
+Dir.glob($path) {|twb| process twb }
+
+emit "\n\n== Done ==\n "

data/lib/twb/dashboard.rb

@@ -22,7 +22,11 @@ module Twb
 
     @@hasher = Digest::SHA256.new
 
-    attr_reader :node, :name, :worksheets, :autosize, :size, :maxheight, :maxwidth, :minheight, :minwidth, :rangesize, :dimensions, :zonecount
+    attr_reader :node
+    attr_reader :name
+    attr_reader :worksheets
+    attr_reader :autosize, :size, :maxheight, :maxwidth, :minheight, :minwidth, :rangesize, :dimensions
+    attr_reader :zonecount
 
     def initialize dashboardNode, twbworksheets
       @node     = dashboardNode

data/lib/twb/datasource.rb

@@ -1,4 +1,4 @@
-#  Copyright (C) 2014, 2015  Chris Gerrard
+#  Copyright (C) 2014, 2015, 2016  Chris Gerrard
 #
 #  This program is free software: you can redistribute it and/or modify
 #  it under the terms of the GNU General Public License as published by
@@ -15,6 +15,7 @@
 
 require 'nokogiri'
 require 'digest/md5'
+require 'json'
 
 module Twb
 
@@ -22,7 +23,25 @@ module Twb
 
     @@hasher = Digest::SHA256.new
 
-    attr_reader :node, :name, :caption, :uiname, :connHash, :dsclass, :connection, :tables, :localfields, :metadatafields
+    @@connGNodeParamsJSON = %q(
+    { "csv"           : { "label" : ["filename"], "id" : ["directory","filename"], "type" : "source" },
+      "excel"         : { "label" : ["filename"], "id" : ["directory","filename"], "type" : "source" },
+      "dataengine"    : { "label" : ["dbname"  ], "id" : ["directory","filename"], "type" : "source" },
+      "msaccess"      : { "label" : ["filename"], "id" : ["directory","filename"], "type" : "source" },
+      "textscan"      : { "label" : ["filename"], "id" : ["directory","filename"], "type" : "source" },
+      "excel-direct"  : { "label" : ["filename"], "id" : [            "filename"], "type" : "source" },
+      "salesforce"    : { "label" : ["server"  ], "id" : [            "server"  ], "type" : "source" }
+    }
+    )
+    @@cgNodeParams  = JSON.parse @@connGNodeParamsJSON
+
+
+    attr_reader :name,        :caption,       :uiname,  :dsclass
+    attr_reader :connection,  :connHash
+    attr_reader :tables      
+    attr_reader :localfields,   :metadatafields, :mappedFields, :fieldUINames, :calculatedFields
+    attr_reader :filters
+    attr_reader :node
 
     def initialize dataSourceNode
       @node    = dataSourceNode
@@ -31,11 +50,22 @@ module Twb
       @uiname  = if @caption.nil? || @caption == '' then @name else @caption end
       processConnection
       processFields
+      processFilters
+      loadTableFields
       return self
     end
 
+    def tableauVersion tabVersion
+      @tableauVersion = tabVersion
+    end
+
+    def tableauVersion
+      @tableauVersion
+    end
+
     def processConnection
       @connection  = @node.at_xpath('./connection')
+      @dsclass     = @name  # handles 'Parameters' data source, which has no connection element (& others if so)  
       unless @connection.nil?
         @dsclass = @connection.attribute('class').text
         # note: must use "dsclass" as "class" would override Rubys ".class" Kernel method
@@ -67,25 +97,27 @@ module Twb
     end
 
     def Parameters?
-      @name == 'Parameters'
+      'Parameters'.eql? @name 
     end
 
     def processFields
       # --
-      @localfields    = {}
-      @metadatafields = {}
+      @localfields      = {}
+      @metadatafields   = {}
+      @calculatedFields = @node.xpath("./column/calculation")
       return if @connection.nil?
       ## load local fields
       connClass    = @node.at_xpath('./connection').attribute('class').text
       fxpath       = case connClass
                      when 'dataengine' then './column'
+                     when 'sqlserver'  then './column'
                      else                   './connection/relation/columns/column'
                      end
       nodes = @node.xpath(fxpath)
       # puts "DATASOURCE ::=>> @node: connClass: '#{connClass.class}' ::: #{connClass.eql?('dataengine')} fxpath: #{fxpath}  :: #{nodes.length}"
       nodes.each do |node|
         field = Twb::LocalField.new(node)
-        @localfields[field.name] = field
+        @localfields[field.dbname] = field
       end
       ## load metadata fields
       nodes = @node.xpath("./connection/metadata-records/metadata-record[@class='column']")
@@ -95,8 +127,166 @@ module Twb
         field = Twb::MetadataField.new(node)
         @metadatafields[field.name] = field
       end
+      # load calculated fields
+      # @calculatedFields = if 'Parameters'.eql? @name
+      #                        @node.xpath("./column/calculation")
+      #                     else
+      #                        @node.xpath(".//column/calculation")
+      #                     end
+    end
+
+    def fieldUIName fieldName
+      loadFieldUINames if @fieldUINames.nil?
+      @fieldUINames[fieldName]
+    end
+
+    def fieldUINames
+      loadFieldUINames if @fieldUINames.nil?
+      return @fieldUINames
+    end
+
+    def loadFieldUINames
+      @fieldUINames = {}
+      columnNodes = @node.xpath('./column')
+      columnNodes.each do |cn|
+        name    = cn.attribute('name').text.gsub(/^\[|\]$/,'')
+        caption = cn.attribute('caption')
+        uiName  = caption.nil? ? name : caption.text
+        @fieldUINames[name]         = uiName
+        @fieldUINames[caption.text] = caption.text  unless caption.nil?    # in case of unintended usage: lookup by Caption name
+      end
+      mappedFields.each do |fname,fmap|
+        dbTable = fmap['table']
+        dbName  = fmap['dbName']
+        @fieldUINames[dbName] = fname  unless @fieldUINames.include?(dbName)
+        @fieldUINames[fname]  = fname  unless @fieldUINames.include?(fname)
+      end
+    end
+
+    def mappedFields
+      return @mappedFields unless @mappedFields.nil?
+      loadTableFields
+      return @mappedFields
+    end
+
+    def fieldTable fieldName
+      loadTableFields if @mappedFields.nil?
+      tf = @mappedFields[fieldName]
+      return tf.nil? ? nil : tf['table']
+    end
+
+    # fields are unique in the data source by UI name
+    def loadTableFields
+      # puts "DATA SOURCE FIELD TABLE LOAD"
+      @mappedFields = {}
+      fieldNodes = @node.xpath('./connection/cols/map')
+      fieldNodes.each do |fn|
+        uiName = fn.attribute('key').text.gsub(/^\[|\]$/,'')
+        fldRef = fn.attribute('value').text.gsub(/^\[|\]$/,'')
+        parts  = fldRef.split('].[')
+        table  = parts[0]
+        dbName = parts[1]
+        @mappedFields[uiName] = {'table' => table, 'dbName' => dbName}
+        # puts "=== #{uiName} :: T=#{@mappedFields[uiName]['table']}  DBN=#{@mappedFields[uiName]['dbName']} "
+      end
+      relTableNodes = @node.xpath('.//relation[@table]')
+      relTableNodes.each do |relNode|
+        table = relNode.attribute('name').text
+        cols = relNode.xpath('./columns/column')
+        cols.each do |col|
+          fldName = col.attribute('name')
+          @mappedFields[fldName.text] = {'table' => table, 'dbName' => @uiname} unless fldName.nil?
+        end
+      end
     end
 
+=begin
+  <filter class='categorical' column='[enforcement_type]' filter-group='2'>
+    <groupfilter function='member' level='[enforcement_type]' member='&quot;towing&quot;' user:ui-domain='database' user:ui-enumeration='inclusive' user:ui-marker='enumerate' />
+  </filter>
+=end
+    def processFilters
+      if @filters.nil?
+        @filters = {}
+        fnodes = @node.xpath('./filter')
+        fnodes.each do |fn|
+          columnN = fn.attribute('column')
+          unless columnN.nil?
+            memberNodes = fn.xpath('.//groupfilter/@member')
+            unless memberNodes.nil?
+              members = []
+              memberNodes.each do |m|
+                members.push m.text
+              end
+              @filters[columnN.text] = members
+            end
+          end 
+        end
+      end
+    end
   end
+  
+  # Generates and returns a set of graph node-edge-node triplets.
+  # The intention is to create a graph that can be standalone used as a subgraph by the function's caller.
+  # The initial implementation only considers a linear list of node names as input, resulting in 
+  # the creation of a single-path structure, e.g.
+  #  [inNode] -> node1 -> node2 -> ... -> nodeN
+  # this may manifest as a tree when there are siblings at any level, e.g.
+  #  [inNode] -> node1 -> node11 -> ... -> nodeX
+  #                    -> node22 -> ... -> nodeY
+  #           -> node2 -> node21 -> ... -> nodeZ
+  # Params:
+  # +inNodes+::   the node(s) to which this function's generated graph is to be linked, may be nil:
+  #   + , or multiple)
+  # +nodesList+:: list of node types by name to be built and linked together, in the order named  
+  def graphNodes nodesList
+    graph     = Twb::Util::Graphedges.new()
+    nodesList.each do |nName|
+    end
+    case @dsclass
+      when 'federated'
+        graph = processFederatedSource nodesList
+    end
+    return graph
+  end
+
+  def processFederatedSource nodesList
+    emit false, "       (federated)  #{dataSource.uiname}"
+    dsGNode     = Twb::Util::Graphnode.new(name: @uiname, id: @name, type: 'Data Connection')
+    graph       = Twb::Util::Graphedges.new(dsGNode)
+    edges       = []
+    dsNode      = dataSource.node
+    connections = dsNode.xpath('./connection/named-connections/named-connection/connection')
+    connections.each do |conn|
+      connClass  = conn.attribute('class').text
+      emit true, "CONN CLASS: #{connClass}"
+      # -- Generating Source Node
+      cgParams = @@cgNodeParams[connClass]
+      # emit true, "cgparams  : #{cgParams}"
+      cLabel       = buildConnGraphPart( conn, cgParams['label'])
+      cID          = buildConnGraphPart( conn, cgParams['id'])
+      cType        = cgParams['type']
+      # emit true, "   label  : #{cLabel}"
+      # emit true, "      id  : #{cID}"
+      # emit true, "    type  : #{cType}"
+      srcNode   = Twb::Util::Graphnode.new(name: cLabel,  id: cID,     type: 'Data Source')
+      graphEdge = Twb::Util::Graphedge.new(from: dsGNode, to: srcNode, relationship: 'is located at')
+      graph.edges << graphEdge
+    end
+    return graph
+  end
+
+  def buildConnGraphPart connNode, attributes
+    return connNode.attribute(attributes) if attributes.is_a? String
+    emit false, "ATTRIBUTES :: #{attributes}"
+    str = ''
+    attributes.each do |attName|
+      attrib = connNode.attribute(attName)
+      emit false, " -#{attName}\t-> #{attrib}"
+      str   += attrib.text unless attrib.nil?
+    end
+    return str
+  end
+
 
 end

data/lib/twb/fieldcalculation.rb

@@ -19,38 +19,74 @@ module Twb
 
   class FieldCalculation
 
-    attr_reader :node, :formula, :fields, :comments, :class
+    attr_reader :node,    :fieldNode
+    attr_reader :formula, :formulaLines,  :formulaFlat
+    attr_reader :uiname,  :techname,      :caption
+    attr_reader :class,   :scopeIsolation
+    attr_reader :fields,  :resolvedFields 
+    attr_reader :comments
 
     def initialize calcNode
-      #puts "CALCNODE:: '#{calcNode}' -> #{calcNode.class}"
       if calcNode
-        @node       = calcNode
+        @node         = calcNode
+        @fieldNode    = @node.xpath('..')
+        # field names
+        @caption,     = attribText(@fieldNode, 'caption')
+        @techname     = @fieldNode.attribute('name').text.gsub(/^\[|\]$/,'')   # assumes the name attribute exists
+        @uiname       = @caption.nil? ? @techName : @caption
        #-- Calculation --
-        formulaCode     = calcNode.xpath('./@formula').text.gsub(/\r\n/, ' ')
-        formulaLines    = calcNode.xpath('./@formula').text.split(/\r\n/)
-        @formula        = getFormula(  formulaLines )
-        @comments       = getComments( formulaLines )
-        @class          = calcNode.xpath('./@class').text
-        @scopeIsolation = calcNode.xpath('./@scope-isolation').text
+        @has_formula    = @node.has_attribute?('formula')
+        if @has_formula
+          @formula      = @node.attribute('formula').text
+          @formulaLines = formula.split(/\n|\r\n/)
+          @formulaFlat  = flattenFormula(formulaLines)
+          @comments     = getComments(formulaLines)
+        end
+        @class          = attribText(@node, 'class')           
+        @scopeIsolation = attribText(@node, 'scope-isolation') 
        #-- Fields      --
-        @fields   = parseFieldsFromFormula(formulaCode)
+        parseFormFields # establishes @ fields
+        resolveFields   # establishes @ resolvedFields
       end
     end
+
+    def attribText(node, attribute)
+      node.attribute(attribute).nil? ? nil : node.attribute(attribute).text
+    end
+
     
-    def parseFieldsFromFormula formula
-      fieldSet = Set.new []
-      if formula =~ /\[.+\]/ then
-          stripFrt =  formula.gsub( /^[^\[]*[\[]/    , '['      )
+    def parseFormFields
+      @fields = Set.new []
+      formula = formulaFlat
+      if formula.include?('[') && formula.include?(']')
+          noSqLits =  formula.gsub(/'[\[\.\]]+'/, ' ')
+          flatForm = noSqLits.gsub( /\n/, ' ')
+          stripFrt = flatForm.gsub( /^[^\[]*[\[]/    , '['      )
           stripBck = stripFrt.gsub( /\][^\]]+$/      , ']'      )
           stripMid = stripBck.gsub( /\][^\]]{2,}\[/  , ']]..[[' )
           stripCom = stripMid.gsub( /\][ ]*,[ ]*\[/  , ']]..[[' )
-          fields   = stripCom.split(']..[')
-          fields.each { |field| fieldSet.add field}
+          stripFns = stripMid.gsub( /\][ ]*[\*\/+\-,][ ]*\[/  , ']]..[[' )
+          fields   = stripFns.split(']..[')
+          fields.each { |field| @fields.add field.gsub(/^\[|\]$/, '')}
+      end
+    end
+
+
+    def resolveFields
+      @resolvedFields = []
+      fields.each do |field|
+        rawField = field.gsub(/^\[|\]$/,'')
+        parts    = rawField.split('].[')
+        if parts.length > 1
+          hash = { :field => parts[1], :source => parts[0] }
+        else
+          hash = { :field => parts[0], :source => nil      }
+        end
+        @resolvedFields << hash unless hash.nil?
       end
-      return fieldSet
     end
 
-    def getFormula lines
+    def flattenFormula lines
       formula = ''
       lines.each do |line|
           line.strip