twb 0.5.3 → 1.0

data/lib/twb/findDSFields.rb

@@ -0,0 +1,153 @@
+require 'nokogiri'
+require 'csv'
+require 'twb'
+
+def init
+  system 'cls'
+  $pFile = File.open('identifyFields.txt','w')
+
+  sqiCSV = 'TWBFieldsByType.csv'
+  $fieldsCSV = CSV.open(sqiCSV, 'w')
+  $fieldsCSV    << [
+                    'TWB', 
+                    'Data Connection - UI', 
+                    'Field Name', 
+                    'TWB Field Type', 
+                    'Data Type', 
+                    'Calc Field?', 
+                   ]
+  $path    = if ARGV.empty? then '**/*.twb' else ARGV[0] end
+  emit " "
+  emit " Identifying fields in Tableau Workbook Data Connections (TWDCs).\n "
+  emit " Looking for Workbooks matching: '#{$path}'  - from: #{ARGV[0]}\n\n "
+end
+
+$localEmit = true
+def emit stuff
+    $pFile.puts stuff
+           puts stuff if $localEmit
+end
+
+$paths = { 
+          'Relation Column' => { 'fieldXPath' => './connection/relation/columns/column',
+                                 'nameXPath'  => './@name' 
+                               },
+          'Metadata Record' => { 'fieldXPath' => './connection/metadata-records/metadata-record[@class="column"]',
+                                 'nameXPath'  => './remote-name[text()]'
+                               },
+          'Columns'         => { 'fieldXPath' => './column',
+                                 'nameXPath'  => './@name' 
+                               },
+         }
+
+# different connection classes have 
+# different paths & attributes to the Relation fields
+$relClassPaths = { 
+                   'dataengine'   => { 'fieldXPath' => './connection/relation/columns/map',
+                                       'nameXPath'  => './@name'
+                                     },
+                   'excel-direct' => { 'fieldXPath' => './connection/relation/columns/map',
+                                       'nameXPath'  => './@name'
+                                     },
+                   'textscan'     => { 'fieldXPath' => './connection/relation/columns/map',
+                                       'nameXPath'  => './@name'
+                                     },
+                  }
+
+def relationFields dataSource, connClass
+  emit "\t relationFields  connClass: '%s'" % [connClass]
+  xPaths     = $relClassPaths[connClass]
+  emit "\t xPaths: #{xPaths}"
+  fnodesPath = xPaths['fieldXPath']
+  fnamePath  = xPaths['nameXPath']
+  fieldNodes = dataSource.xpath(fnodesPath)
+    
+end
+
+def metaDataRecords dataSource
+  emit "\t metaDataRecords  " #connClass: '%s'" % [connClass]
+  mdNodes = dataSource.xpath('./connection/metadata-records/metadata-record[@class="column"]')
+  emit "  mdNodes.size: #{mdNodes.size}"
+  fields = {}
+  mdNodes.each do |node|
+    name = node.xpath('./local-name').text.gsub(/^\[/,'').gsub(/\]$/,'')
+    type = node.xpath('./local-type').text
+    fields[name] = {'type' => type}
+  end
+  return fields
+end
+
+def localColumns dataSource
+  emit "\t localColumns  " #connClass: '%s'" % [connClass]
+  colNodes = dataSource.xpath('./column')
+  emit "  colNodes.size: #{colNodes.size}"
+  fields = {}
+  colNodes.each do |node|
+    calcField = !node.at_xpath('./calculation').nil?
+    caption   = node.attribute('caption')
+    name      = node.attribute('name').text.gsub(/^\[/,'').gsub(/\]$/,'')
+    type      = node.attribute('datatype')
+    emit "  \t    name: %s\n \t    type: %s\n \t caption: %s" % [name, type, caption]
+    caption   = name if caption.nil?
+    emit "  \t caption: %s  \n " % [caption]
+    fields[caption] = {'type' => type, 'calcField' => calcField}
+  end
+  return fields
+end
+
+def process file
+  emit "\n == #{file}"
+  doc = Nokogiri::XML(open(file))
+  dataSourcesNodes = doc.xpath('//workbook/datasources/datasource')
+  dataSourcesNodes.each do |ds|
+    dsCaption = ds.attribute('caption')
+    dsName    = ds.attribute('name')
+    emit "\n  dsCapt: #{dsCaption}"
+    emit   "  dsName: '#{dsName}'\n   ---"
+    connClass = ds.xpath('./connection/@class')
+    emit   "dsCClass: '#{connClass}'\n   ---"
+    if dsName != 'Parameters' then
+      # relationFields  ds, connClass
+      mdFields = metaDataRecords ds
+      mdFields.each do |fname, payload|
+        emit "\t "
+        $fieldsCSV << [file, dsCaption, fname, 'metadata', payload['type'] ]
+        emit  [file, dsCaption, fname, 'metadata', payload['type'] ]
+      end
+      colFields = localColumns  ds
+      colFields.each do |fname, payload|
+        emit "\t "
+        $fieldsCSV << [file, dsCaption, fname, 'column', payload['type'], payload['calcField'] ]
+        emit          [file, dsCaption, fname, 'column', payload['type'], payload['calcField'] ]
+      end
+    end
+
+#    $paths.each do |type, paths|
+#      emit "    type: #{type}"
+#      emit paths['fieldXPath']
+#      nodes = ds.xpath(paths['fieldXPath']).to_a
+#      nodes.each do |node|
+#          name = node.xpath(paths['nameXPath']).text
+#          emit "   fname: #{name}"
+#      end
+#    end
+  end
+end
+
+init
+
+def showPaths
+  $paths.each do |type, paths|
+  	emit "  #{type}"
+    paths.each do |key, value|
+      emit "      : %10s  =>  %-s" % [key, value]    
+    end
+    puts " "
+  end
+end
+
+emit "XXX \n#{$relClassPaths}"
+
+Dir.glob($path) {|twb| process twb }
+
+emit "\n\n== Done ==\n "

data/lib/twb/graph.rb

@@ -0,0 +1,53 @@
+#  Copyright (C) 2014, 2017  Chris Gerrard
+#
+#  This program is free software: you can redistribute it and/or modify
+#  it under the terms of the GNU General Public License as published by
+#  the Free Software Foundation, either version 3 of the License, or
+#  (at your option) any later version.
+#
+#  This program is distributed in the hope that it will be useful,
+#  but WITHOUT ANY WARRANTY; without even the implied warranty of
+#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#  GNU General Public License for more details.
+#
+#  You should have received a copy of the GNU General Public License
+#  along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+module Twb
+
+  class Graph
+
+    # @root - the visible name
+    # @id   - the technical identifier, used to distinquish the node from similarly named nodes
+    # @type - useful for categorizing the node
+    attr_reader   :name, :id, :type
+    attr_accessor :properties
+
+
+    def initialize (name:, id:, type:, properties: {})
+      @name        = name
+      @id          = id
+      @type        = type
+      @properties  = properties
+    end
+
+    def dotLabel
+      # "JIRA 1::JIRA 1.csv" [label="JIRA 1.csv"]
+      "\"%s\" [label=\"%s\"]" % [id, name] 
+    end
+
+    def eql? other
+      @name == other.name && @id == other.id && @type == other.type  && @properties == other.properties
+    end
+
+    def hash
+      [@name, @id, @type, @properties].hash
+    end
+
+    def to_s
+      "name:'%s'  id:'%s'  t:'%s'  p:'%s'" % [@name, @id, @type, @properties.to_s]
+    end
+
+  end
+
+end

data/lib/twb/graphedges.rb

@@ -0,0 +1,36 @@
+#  Copyright (C) 2014, 2017  Chris Gerrard
+#
+#  This program is free software: you can redistribute it and/or modify
+#  it under the terms of the GNU General Public License as published by
+#  the Free Software Foundation, either version 3 of the License, or
+#  (at your option) any later version.
+#
+#  This program is distributed in the hope that it will be useful,
+#  but WITHOUT ANY WARRANTY; without even the implied warranty of
+#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#  GNU General Public License for more details.
+#
+#  You should have received a copy of the GNU General Public License
+#  along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+module Twb
+
+  class Graphedges
+
+    # @root   - the graph's root node
+    # @edges  - the collection of the graph's edges
+    attr_reader   :root
+    attr_accessor :edges
+
+    def initialize (root:)
+      @root        = root if root.instance_of? Twb::Graphnode
+      @edges       = []
+    end
+
+   def to_s
+      "Root:'%s'  edges::%s::" % [@root.to_s, @edges.to_s]
+    end
+
+  end
+
+end

data/lib/twb/graphnode.rb

@@ -0,0 +1,53 @@
+#  Copyright (C) 2014, 2017  Chris Gerrard
+#
+#  This program is free software: you can redistribute it and/or modify
+#  it under the terms of the GNU General Public License as published by
+#  the Free Software Foundation, either version 3 of the License, or
+#  (at your option) any later version.
+#
+#  This program is distributed in the hope that it will be useful,
+#  but WITHOUT ANY WARRANTY; without even the implied warranty of
+#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#  GNU General Public License for more details.
+#
+#  You should have received a copy of the GNU General Public License
+#  along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+module Twb
+
+  class Graphnode
+
+    # @name - the visible name
+    # @id   - the technical identifier, used to distinquish the node from similarly named nodes
+    # @type - useful for categorizing the node
+    attr_reader   :name, :id, :type
+    attr_accessor :properties
+
+
+    def initialize (name:, id:, type:, properties: {})
+      @name        = name
+      @id          = id
+      @type        = type
+      @properties  = properties
+    end
+
+    def dotLabel
+      # "JIRA 1::JIRA 1.csv" [label="JIRA 1.csv"]
+      "\"%s\" [label=\"%s\"]" % [id, name] 
+    end
+
+    def eql? other
+      @name == other.name && @id == other.id && @type == other.type  && @properties == other.properties
+    end
+
+    def hash
+      [@name, @id, @type, @properties].hash
+    end
+
+    def to_s
+      "name:'%s'  id:'%s'  t:'%s'  p:'%s'" % [@name, @id, @type, @properties.to_s]
+    end
+
+  end
+
+end

data/lib/twb/identifyFields.rb

@@ -0,0 +1,103 @@
+require 'nokogiri'
+require 'csv'
+require 'twb'
+
+def init
+  system 'cls'
+  $pFile = File.open('identifyFields.txt','w')
+
+  sqiCSV = 'TWBFieldsByType.csv'
+  $fieldsCSV = CSV.open(sqiCSV, 'w')
+  $fieldsCSV    << [
+                    'TWB', 
+                    'Data Connection - UI', 
+                    'Field Name', 
+                    'Field Type', 
+                   ]
+  $path    = if ARGV.empty? then '**/*.twb' else ARGV[0] end
+  emit " "
+  emit " Identifying fields in Tableau Workbook Data Connections (TWDCs).\n "
+  emit " Looking for Workbooks matching: '#{$path}'  - from: #{ARGV[0]}\n\n "
+end
+
+def method_name
+    datasources     = {}
+    dataSourcesNode  = doc.at_xpath('//workbook/datasources')
+    dataSourcesNodes = doc.xpath('//workbook/datasources/datasource').to_a
+    puts "  dsn: #{dataSourcesNodes}"
+    datasourceNodes.each do |node|
+        datasource = Twb::DataSource.new(node)
+        datasources[datasource.name] = datasource
+    end
+end
+  
+$localEmit = true
+def emit stuff
+    $pFile.puts stuff
+           puts stuff if $localEmit
+end
+
+$paths = { 
+          'Relation Column' => { 'fieldXPath' => './connection/relation/columns/column',
+                                 'nameXPath'  => '@name' 
+                               },
+          'Metadata Record' => { 'fieldXPath' => './connection/metadata-records/metadata-record[@class="column"]',
+                                 'nameXPath'  => './remote-name'
+                               },
+          'Columns'         => { 'fieldXPath' => './column',
+                                 'nameXPath'  => '@name' 
+                               },
+         }
+
+def proc file
+  emit "\n == #{file}"
+end
+
+def process file
+  emit "\n == #{file}"
+  doc = Nokogiri::XML(open(file))
+  dataSourcesNodes = doc.xpath('//workbook/datasources/datasource')
+  dataSourcesNodes.each do |ds|
+    emit "\n   dc: #{ds.attribute('caption')}"
+    emit   "   dn: #{ds.attribute('name')}\n   ---"
+    typeCounts = {}
+    $paths.each do |type, path|
+      nodes = ds.xpath(path).to_a
+#      emit "     : %3i  %-17s  %-s" % [nodes.size, type, path]
+      typeCnt = nodes.size
+      nodes.each do |n|
+#        $fieldsCSV << [file,ds.attribute('name'),n.attribute('name'),type]
+      end
+      if type == 'Columns' then 
+        calcCnt = 0
+        nodes.each do |n|
+          calc = n.xpath('./calculation')
+#          emit "    c:   #{calc.size} "
+          calcCnt += calc.size
+        end
+#        emit '   ---'
+#        emit "   c#: %3i " % [calcCnt]
+#        emit "  !c#: %3i " % [nodes.size - calcCnt]
+        typeCounts['Columns']            = nodes.size 
+        typeCounts['Columns - calc']     = calcCnt 
+        typeCounts['Columns - not calc'] = nodes.size - calcCnt 
+      else
+        typeCounts[type] = typeCnt 
+      end
+    end
+#    emit '   ---'
+    typeCounts.each do |t,c|
+      emit "   tc: %3i  %-s" % [c, t]
+    end
+  end
+end
+
+init
+
+$paths.each do |path|
+	emit "  p: #{path}"
+end
+
+Dir.glob($path) {|twb| process twb }
+
+emit "\n\n== Done ==\n "

data/lib/twb/localfield.rb

@@ -21,12 +21,13 @@ module Twb
 
   class LocalField
 
-    attr_reader :type, :node, :name, :datatype, :role, :type, :hidden, :caption, :aggregation, :uiname, :calculation, :comments
+    attr_reader :type, :node, :tableauname, :dbname, :datatype, :role, :type, :hidden, :caption, :aggregation, :uiname, :calculation, :comments
 
     def initialize fieldNode
       @node        = fieldNode
       @type        = 'local'
-      @name        = @node.attr('name')
+      @tableauname = @node.attr('name')
+      @dbname      = @node.attr('name').gsub(/^\[/,'').gsub(/\]$/,'')
       @datatype    = @node.attr('datatype')
       @role        = @node.attr('role')
       @type        = @node.attr('type')
@@ -35,15 +36,15 @@ module Twb
       @aggregation = @node.attr('aggregation')
       @calculation = getCalculation
       @comments    = getComments
-      @uiname      = if @caption.nil? || @caption == '' then @name.gsub(/^\[/,'').gsub(/\]$/,'') else @caption end
+      @uiname      = if @caption.nil? || @caption == '' then @dbname else @caption end
       return self
     end
-    
+
     def getCalculation
       calcNode = @node.at_xpath("./calculation")
       FieldCalculation.new(calcNode) unless calcNode.nil?
     end
-    
+
     def getComments
       comments = ''
       runs = node.xpath('./desc/formatted-text/run')
@@ -54,7 +55,7 @@ module Twb
       end
       return comments
     end
-    
+
     def remove_attribute attribute
       @node.remove_attribute(attribute)
     end

data/lib/twb/util/graphedge.rb

@@ -0,0 +1,69 @@
+#  Copyright (C) 2017  Chris Gerrard
+#
+#  This program is free software: you can redistribute it and/or modify
+#  it under the terms of the GNU General Public License as published by
+#  the Free Software Foundation, either version 3 of the License, or
+#  (at your option) any later version.
+#
+#  This program is distributed in the hope that it will be useful,
+#  but WITHOUT ANY WARRANTY; without even the implied warranty of
+#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#  GNU General Public License for more details.
+#
+#  You should have received a copy of the GNU General Public License
+#  along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+module Twb
+module Util
+
+  class Graphedge
+
+    # @from - the origin node
+    # @to   - the destination node
+    # @relationship - useful for categorizing the edge
+    # @properties   - useful for categorizing the edge
+    attr_reader   :from, :to, :relationship
+    attr_accessor :properties
+    attr_reader   :cypherCreate
+
+    # Neo4J cypher variable quote character: `  
+
+    def initialize (from:, to:, relationship:, properties: {})
+      raise ArgumentError.new("from: parameter must be a Graphnode, is a '#{from.class}'") unless from.is_a? Twb::Util::Graphnode
+      raise ArgumentError.new(  "to: parameter must be a Graphnode, is a '#{to.class}'"  ) unless   to.is_a? Twb::Util::Graphnode
+      @from         = from
+      @to           = to
+      @relationship = relationship
+      @properties   = properties
+      @cypherCreate = "CREATE  #{cypher_s}"
+    end
+
+    def eql? other
+      @from == other.from && @to == other.to && @relationship == other.relationship && @properties == other.properties
+    end
+
+    def hash
+      [@from.hash, @to.hash, @relationship, @properties].hash
+    end
+
+    def to_s
+      "'#{@from.name}//{@from.id}' --#{@relationship}--> '#{@to.name}//#{@to.id}'"
+    end
+
+    def dot
+      "%s  ->  %s" % [dotquote(from.id), dotquote(to.id)]
+    end
+
+    def dotquote str
+      ns = str.gsub(/(["])/,'\\"')
+      return "\"#{ns}\""
+    end
+
+    def cypher_s
+      "(%s)-[:`%s`]->(%s)" % [@from.cypherID,@relationship,@to.cypherID]
+    end
+
+  end
+
+end  # module Util
+end  # module Twb

data/lib/twb/util/graphedges.rb

@@ -0,0 +1,38 @@
+#  Copyright (C) 2014, 2017  Chris Gerrard
+#
+#  This program is free software: you can redistribute it and/or modify
+#  it under the terms of the GNU General Public License as published by
+#  the Free Software Foundation, either version 3 of the License, or
+#  (at your option) any later version.
+#
+#  This program is distributed in the hope that it will be useful,
+#  but WITHOUT ANY WARRANTY; without even the implied warranty of
+#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#  GNU General Public License for more details.
+#
+#  You should have received a copy of the GNU General Public License
+#  along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+module Twb
+module Util
+
+  class Graphedges
+
+    # @root   - the graph's root node
+    # @edges  - the collection of the graph's edges
+    attr_reader   :root
+    attr_accessor :edges
+
+    def initialize (root = nil)
+      @root        = root if root.instance_of? Twb::Util::Graphnode
+      @edges       = []
+    end
+
+   def to_s
+      "Root:'%s'  edges::%s::" % [@root.to_s, @edges.to_s]
+    end
+
+  end
+
+end  # module Util
+end  # module Twb

data/lib/twb/util/graphnode.rb

@@ -0,0 +1,94 @@
+#  Copyright (C) 2014, 2017  Chris Gerrard
+#
+#  This program is free software: you can redistribute it and/or modify
+#  it under the terms of the GNU General Public License as published by
+#  the Free Software Foundation, either version 3 of the License, or
+#  (at your option) any later version.
+#
+#  This program is distributed in the hope that it will be useful,
+#  but WITHOUT ANY WARRANTY; without even the implied warranty of
+#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#  GNU General Public License for more details.
+#
+#  You should have received a copy of the GNU General Public License
+#  along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+module Twb
+module Util
+
+  class Graphnode
+ 
+    @@stripChars = /[.: %-\-\(\)=]/
+    @@replChar   = '_'
+
+
+
+    # @name - the visible name
+    # @id   - the technical identifier, used to distinquish the node from similarly named nodes
+    # @type - useful for categorizing the node
+    attr_reader   :id, :type, :name
+    attr_reader   :cypherID, :cypherCreate
+  # attr_reader   :cypherName, :cypherID, :cypherNodeID, :cypherType, :cypherCreate
+    attr_accessor :properties
+
+    # Neo4J cypher variable quote character: `  
+
+    def initialize (name:, id:, type:, properties: {})
+      @id           = id
+      @type         = type
+      @name         = name
+      @properties   = properties
+      # --
+      @cypherID     = '`' + id + '`'
+      # @cypherType   = type
+      # @cypherName   = name
+      @properties['name'] = name unless @properties.key? 'name'
+      @cypherCreate = "CREATE  (%s:`%s` {%s})" % [@cypherID,type,props_s]
+    end
+
+
+    def cypherize str
+      enquote str.gsub(@@stripChars,@@replChar)
+    end
+
+    def enquote str
+      str.gsub("'","\\\\'")
+    end
+
+
+    def dotLabel
+      # "JIRA 1::JIRA 1.csv" [label="JIRA 1.csv"]
+      filled = @type =~ /Data Source|DB Table|Database Field/ ? "style=filled" : ''
+      "\"%s\" [label=\"%s\"  %s]" % [id, name, filled] 
+    end
+
+
+    def eql? other
+      @name == other.name && @id == other.id && @type == other.type  && @properties == other.properties
+    end
+
+
+    def hash
+      [@name, @id, @type, @properties].hash
+    end
+
+
+    def to_s
+      "name:'%s'  id:'%s'  t:'%s'  p:%s" % [@name, @id, @type, @properties.to_s]
+    end
+
+
+    def cypher_s
+      "name:%s  id:%s  t:%s  p:{%s}" % [name, @cypherID, @type, props_s]
+    end
+
+
+    def props_s
+      @properties.map{|k,v| "#{k}: #{v.inspect}"}.join(', ')
+    end
+
+
+  end
+
+end  # module Util
+end  # module Twb