twb 1.0.5 → 1.9.1

data/lib/twb/dashboard.rb

@@ -65,7 +65,7 @@ module Twb
       @minheight  = size.attr('minheight')
       @minwidth   = size.attr('minwidth')
       @rangesize  = size.attr('rangesize')
-      @dimensions = @minwidth.to_s + ':' + @minheight.to_s + ':' +@maxwidth.to_s + ':' +@maxheight.to_s
+      @dimensions = "%s:%s:%s:%s" % [@minwidth.to_s, @minheight.to_s, @maxwidth.to_s, @maxheight.to_s]
     end
 
   end

data/lib/twb/datasource.rb

@@ -20,36 +20,47 @@ require 'json'
 module Twb
 
   class DataSource
-
+  
     @@hasher = Digest::SHA256.new
 
     @@connGNodeParamsJSON = %q(
-    { "csv"           : { "label" : ["filename"], "id" : ["directory","filename"], "type" : "source" },
-      "excel"         : { "label" : ["filename"], "id" : ["directory","filename"], "type" : "source" },
-      "dataengine"    : { "label" : ["dbname"  ], "id" : ["directory","filename"], "type" : "source" },
-      "msaccess"      : { "label" : ["filename"], "id" : ["directory","filename"], "type" : "source" },
-      "textscan"      : { "label" : ["filename"], "id" : ["directory","filename"], "type" : "source" },
-      "excel-direct"  : { "label" : ["filename"], "id" : [            "filename"], "type" : "source" },
-      "salesforce"    : { "label" : ["server"  ], "id" : [            "server"  ], "type" : "source" }
+    { "csv"           : { "label" : ["filename"], "id" : ["directory","filename"], "type" : "CSV"        },
+      "excel"         : { "label" : ["filename"], "id" : ["directory","filename"], "type" : "Excel"      },
+      "dataengine"    : { "label" : ["dbname"  ], "id" : ["directory","filename"], "type" : "TDE"        },
+      "msaccess"      : { "label" : ["filename"], "id" : ["directory","filename"], "type" : "MS Access"  },
+      "oracle"        : { "label" : ["server"  ], "id" : [            "server"  ], "type" : "Oracle"     },
+      "postgres"      : { "label" : ["server"  ], "id" : [            "server"  ], "type" : "PostgreSQL" },
+      "textscan"      : { "label" : ["filename"], "id" : ["directory","filename"], "type" : "CSV / TSV"  },
+      "excel-direct"  : { "label" : ["filename"], "id" : [            "filename"], "type" : "Excel"      },
+      "salesforce"    : { "label" : ["server"  ], "id" : [            "server"  ], "type" : "Salesforce" }
     }
     )
     @@cgNodeParams  = JSON.parse @@connGNodeParamsJSON
 
 
-    attr_reader :name,        :caption,       :uiname,  :dsclass
-    attr_reader :connection,  :connHash
-    attr_reader :tables      
-    attr_reader :localfields, :metadatafields, :mappedFields, :fieldUINames, :calculatedFields
+    attr_reader :workbook
+    attr_reader :name,          :caption,         :uiname
+    attr_reader :dsclass,       :isExtract
+    attr_reader :connection,    :connHash
+    attr_reader :tables,        :joinPairs      
+    attr_reader :localFields,   :localFieldNames, :localField, :hasField 
+    attr_reader :columnFields 
+    attr_reader :metadataFields 
+    attr_reader :dbFields
+    attr_reader :mappedFields
+    attr_reader :fieldUINames
+    attr_reader :calculatedFields, :calculatedFieldNamesMap, :calculatedFieldNames, :calculatedField 
+    attr_reader :allFields 
     attr_reader :filters
     attr_reader :node
 
-    def initialize dataSourceNode
-      @node    = dataSourceNode
-      @name    = @node.attr('name')
-      @caption = @node.attr('caption')
-      @uiname  = if @caption.nil? || @caption == '' then @name else @caption end
+    def initialize dataSourceNode, workbook
+      @node     = dataSourceNode
+      @workbook = workbook 
+      @name     = @node.attr('name')
+      @caption  = @node.attr('caption')
+      @uiname   = if @caption.nil? || @caption == '' then @name else @caption end
       processConnection
-      processFields
       processFilters
       loadTableFields
       return self
@@ -88,6 +99,10 @@ module Twb
       @connHash = Digest::MD5.hexdigest(dsConnStr)
     end
 
+    def isExtract
+      @isExtract ||= @isExtract = !@node.at_xpath('./extract').nil?
+    end
+
     def loadTables connection
       @tables = {}
       nodes = connection.xpath(".//relation[@type='table']")
@@ -96,43 +111,115 @@ module Twb
       end
     end
 
+    def joinPairs
+      @joinPairs ||= loadJoinPairs
+    end
+
+    def loadJoinPairs
+      @joinPairs = Set.new
+      mainJoin   = @node.xpath("./connection/relation[@type='join']")
+      clauses    = @node.xpath(".//relation[@type='join']/clause") 
+      clauses.each do |clause|
+        leafs = clause.xpath('.//expression[not(node())]')
+        @joinPairs << [ pullTable(leafs[0]) , pullTable(leafs[1]) ]
+      end
+      return @joinPairs
+    end
+
+    def joinTree
+      @joinTree ||= loadJoinTree
+    end
+
+    def loadJoinTree
+      loadJoinPairs if @joinPairs.nil?
+      # puts "LJT::#{@uiname}::joinPairs:: #{@joinPairs.inspect}"
+      # @joinPairs.each { |jp| puts "JP::#{jp}" }
+      @joinTree = JoinTree.new(@name)
+      @joinPairs.each do |from,to|
+        # puts "from:#{from} -> to:#{to}"
+        tableFrom = JoinTable.new(from)
+        tableTo   = JoinTable.new(to)
+        @joinTree.add(tableFrom, tableTo)
+      end
+      # puts '---'
+      return @joinTree
+    end
+
+    def pullTable xml
+        code =xml.attribute('op').text
+        table = code.split('].[')[0][1..-1]
+    end
+
+
     def Parameters?
       'Parameters'.eql? @name 
     end
 
-    def processFields
-      # --
-      @localfields      = {}
-      @metadatafields   = {}
-      @calculatedFields = @node.xpath("./column/calculation")
-      return if @connection.nil?
-      ## load local fields
-      connClass    = @node.at_xpath('./connection').attribute('class').text
-      fxpath       = case connClass
-                     when 'dataengine' then './column'
-                     when 'sqlserver'  then './column'
-                     else                   './connection/relation/columns/column'
-                     end
-      nodes = @node.xpath(fxpath)
-      # puts "DATASOURCE ::=>> @node: connClass: '#{connClass.class}' ::: #{connClass.eql?('dataengine')} fxpath: #{fxpath}  :: #{nodes.length}"
-      nodes.each do |node|
-        field = Twb::LocalField.new(node)
-        @localfields[field.dbname] = field
+    def columnFields
+      @columnFields ||= loadColumnFields
+    end
+
+    def loadColumnFields
+      @columnFields = Set.new
+      nodes = @node.xpath('./column')
+      nodes.each do |n|
+        field = Twb::ColumnField.new n
+        @columnFields << field
       end
-      ## load metadata fields
-      nodes = @node.xpath("./connection/metadata-records/metadata-record[@class='column']")
-      # note: there are other nodes "<metadata-record class='capability'>" whose nature is unclear
-      #       these nodes have no value for their <name node, so we're not loading them
-      nodes.each do |node|
-        field = Twb::MetadataField.new(node)
-        @metadatafields[field.name] = field
+      return @columnFields
+    end 
+
+    def localFields
+      @localFields ||= loadLocalFields
+    end
+
+    def loadLocalFields
+      @localFields = {}
+      unless @connection.nil?  # Parameters has no connection node, & no local fields
+        connClass    = @node.at_xpath('./connection').attribute('class').text
+        fxpath       = case connClass
+                       when 'dataengine' then './column'
+                       when 'sqlserver'  then './column'
+                       else                   './connection/relation/columns/column'
+                       end
+        nodes = @node.xpath(fxpath)
+        nodes.each do |node|
+          field = Twb::LocalField.new(node)
+          @localFields[field.name] = field
+        end
       end
-      # load calculated fields
-      # @calculatedFields = if 'Parameters'.eql? @name
-      #                        @node.xpath("./column/calculation")
-      #                     else
-      #                        @node.xpath(".//column/calculation")
-      #                     end
+      return @localFields
+    end
+
+    def metadataFields
+      @metadataFields ||= loadMetadataFields
+    end
+
+    def loadMetadataFields
+      @metadataFields = Set.new
+      unless @connection.nil?  # Parameters has no connection node, & no metadata fields
+        # nodes = @node.xpath(".//metadata-record[@class='column']")
+        # # note: there are other nodes "<metadata-record class='capability'>" whose nature is unclear
+        # #       these nodes have no value for their <name node, so are not loaded
+        nodes = @node.xpath('./connection//metadata-record')
+        nodes.each do |node|
+          field = Twb::MetadataField.new(node)
+          field.source = :db
+          @metadataFields << field
+        end
+        nodes = @node.xpath('./extract//metadata-record')
+        nodes.each do |node|
+          field = Twb::MetadataField.new(node)
+          field.source = :extract
+          @metadataFields << field
+        end
+      end
+      return @metadataFields
+    end
+
+    def updateTime
+      attr = @node.xpath('.//@update-time').first
+      @updateTime = attr.nil? ? nil : attr.value
     end
 
     def fieldUIName fieldName
@@ -163,12 +250,58 @@ module Twb
       end
     end
 
-    def mappedFields
+    def dbFields
       return @mappedFields unless @mappedFields.nil?
       loadTableFields
       return @mappedFields
     end
 
+    def mappedFields
+      loadTableFields if @mappedFields.nil?
+      return @mappedFields
+    end
+
+    # NOTE: Calculated Fields are mapped by their UI name
+    def calculatedFieldsMap
+      @calculatedFieldsMap ||= loadCalculatedFields
+    end
+
+    def calculatedFields
+      loadCalculatedFields if @calculatedFieldsMap.nil?
+      return @calculatedFieldsMap.values
+    end
+
+    def calculatedFieldNames
+      loadCalculatedFields if @calculatedFieldsMap.nil?
+      return @calculatedFieldsMap.keys
+    end
+
+    def calculatedField(name)
+      loadCalculatedFields if @calculatedFieldsMap.nil?
+      return @calculatedFieldsMap[name]
+    end
+
+    def loadCalculatedFields
+      @calculatedFieldsMap = {}
+      cfnodes = @node.xpath("./column[calculation]")
+      cfnodes.each do |node|
+        calcField = Twb::CalculatedField.new node, self
+        @calculatedFieldsMap[calcField.uiname] = calcField
+      end
+      return @calculatedFieldsMap
+    end
+
+    def allFields
+      return @allFields unless @allFields.nil?
+      @allFields = SortedSet.new
+      dbf = dbFields
+      @allFields << dbf.keys
+    end
+
+    def has_field? fieldName
+      dbFields.has_key? fieldName
+    end
+
     def fieldTable fieldName
       loadTableFields if @mappedFields.nil?
       tf = @mappedFields[fieldName]
@@ -200,11 +333,11 @@ module Twb
       end
     end
 
-=begin
-  <filter class='categorical' column='[enforcement_type]' filter-group='2'>
-    <groupfilter function='member' level='[enforcement_type]' member='&quot;towing&quot;' user:ui-domain='database' user:ui-enumeration='inclusive' user:ui-marker='enumerate' />
-  </filter>
-=end
+    # =begin
+    #  <filter class='categorical' column='[enforcement_type]' filter-group='2'>
+    #    <groupfilter function='member' level='[enforcement_type]' member='&quot;towing&quot;' user:ui-domain='database' user:ui-enumeration='inclusive' user:ui-marker='enumerate' />
+    #  </filter>
+    # end
     def processFilters
       if @filters.nil?
         @filters = {}
@@ -224,69 +357,183 @@ module Twb
         end
       end
     end
-  end
-  
-  # Generates and returns a set of graph node-edge-node triplets.
-  # The intention is to create a graph that can be standalone used as a subgraph by the function's caller.
-  # The initial implementation only considers a linear list of node names as input, resulting in 
-  # the creation of a single-path structure, e.g.
-  #  [inNode] -> node1 -> node2 -> ... -> nodeN
-  # this may manifest as a tree when there are siblings at any level, e.g.
-  #  [inNode] -> node1 -> node11 -> ... -> nodeX
-  #                    -> node22 -> ... -> nodeY
-  #           -> node2 -> node21 -> ... -> nodeZ
-  # Params:
-  # +inNodes+::   the node(s) to which this function's generated graph is to be linked, may be nil:
-  #   + , or multiple)
-  # +nodesList+:: list of node types by name to be built and linked together, in the order named  
-  def graphNodes nodesList
-    graph     = Twb::Util::Graphedges.new()
-    nodesList.each do |nName|
-    end
-    case @dsclass
-      when 'federated'
-        graph = processFederatedSource nodesList
-    end
-    return graph
-  end
-
-  def processFederatedSource nodesList
-    emit false, "       (federated)  #{dataSource.uiname}"
-    dsGNode     = Twb::Util::Graphnode.new(name: @uiname, id: @name, type: 'Data Connection')
-    graph       = Twb::Util::Graphedges.new(dsGNode)
-    edges       = []
-    dsNode      = dataSource.node
-    connections = dsNode.xpath('./connection/named-connections/named-connection/connection')
-    connections.each do |conn|
-      connClass  = conn.attribute('class').text
-      emit true, "CONN CLASS: #{connClass}"
-      # -- Generating Source Node
-      cgParams = @@cgNodeParams[connClass]
-      # emit true, "cgparams  : #{cgParams}"
-      cLabel       = buildConnGraphPart( conn, cgParams['label'])
-      cID          = buildConnGraphPart( conn, cgParams['id'])
-      cType        = cgParams['type']
-      # emit true, "   label  : #{cLabel}"
-      # emit true, "      id  : #{cID}"
-      # emit true, "    type  : #{cType}"
-      srcNode   = Twb::Util::Graphnode.new(name: cLabel,  id: cID,     type: 'Data Source')
-      graphEdge = Twb::Util::Graphedge.new(from: dsGNode, to: srcNode, relationship: 'is located at')
-      graph.edges << graphEdge
-    end
-    return graph
-  end
-
-  def buildConnGraphPart connNode, attributes
-    return connNode.attribute(attributes) if attributes.is_a? String
-    emit false, "ATTRIBUTES :: #{attributes}"
-    str = ''
-    attributes.each do |attName|
-      attrib = connNode.attribute(attName)
-      emit false, " -#{attName}\t-> #{attrib}"
-      str   += attrib.text unless attrib.nil?
-    end
-    return str
-  end
-
-
-end
+
+  end # class DataSource
+
+  # DataSource Utilities
+
+  # # Generates and returns a set of graph node-edge-node triplets.
+  # # The intention is to create a graph that can be standalone used as a subgraph by the function's caller.
+  # # The initial implementation only considers a linear list of node names as input, resulting in 
+  # # the creation of a single-path structure, e.g.
+  # #  [inNode] -> node1 -> node2 -> ... -> nodeN
+  # # this may manifest as a tree when there are siblings at any level, e.g.
+  # #  [inNode] -> node1 -> node11 -> ... -> nodeX
+  # #                    -> node22 -> ... -> nodeY
+  # #           -> node2 -> node21 -> ... -> nodeZ
+  # # Params:
+  # # +inNodes+::   the node(s) to which this function's generated graph is to be linked, may be nil:
+  # #   + , or multiple)
+  # # +nodesList+:: list of node types by name to be built and linked together, in the order named  
+  # def graphNodes nodesList
+  #   graph     = Twb::Util::Graphedges.new()
+  #   nodesList.each do |nName|
+  #   end
+  #   case @dsclass
+  #     when 'federated'
+  #       graph = processFederatedSource nodesList
+  #   end
+  #   return graph
+  # end
+
+  # def processFederatedSource nodesList
+  #   emit false, "       (federated)  #{dataSource.uiname}"
+  #   dsGNode     = Twb::Util::Graphnode.new(name: @uiname, id: @name, type: 'Data Connection')
+  #   graph       = Twb::Util::Graphedges.new(dsGNode)
+  #   edges       = []
+  #   dsNode      = dataSource.node
+  #   connections = dsNode.xpath('./connection/named-connections/named-connection/connection')
+  #   connections.each do |conn|
+  #     connClass  = conn.attribute('class').text
+  #     emit true, "CONN CLASS: #{connClass}"
+  #     # -- Generating Source Node
+  #     cgParams = @@cgNodeParams[connClass]
+  #     # emit true, "cgparams  : #{cgParams}"
+  #     cLabel       = buildConnGraphPart( conn, cgParams['label'])
+  #     cID          = buildConnGraphPart( conn, cgParams['id'])
+  #     cType        = cgParams['type']
+  #     # emit true, "   label  : #{cLabel}"
+  #     # emit true, "      id  : #{cID}"
+  #     # emit true, "    type  : #{cType}"
+  #     srcNode   = Twb::Util::Graphnode.new(name: cLabel,  id: cID,     type: 'Data Source')
+  #     graphEdge = Twb::Util::Graphedge.new(from: dsGNode, to: srcNode, relationship: 'is located at')
+  #     graph.edges << graphEdge
+  #   end
+  #   return graph
+  # end
+
+  # def buildConnGraphPart connNode, attributes
+  #   return connNode.attribute(attributes) if attributes.is_a? String
+  #   emit false, "ATTRIBUTES :: #{attributes}"
+  #   str = ''
+  #   attributes.each do |attName|
+  #     attrib = connNode.attribute(attName)
+  #     emit false, " -#{attName}\t-> #{attrib}"
+  #     str   += attrib.text unless attrib.nil?
+  #   end
+  #   return str
+  # end
+
+
+  class JoinTablePair
+    attr_reader :from, :to
+    def initialize(from, to)
+      raise ParamaterException.new("'From' cannot be nil.") if from.nil?
+      raise ParamaterException.new("'To' cannot be nil.")   if to.nil?
+      @from = from
+      @to   = to
+    end
+    def to_s
+      "#{@from} -> #{@to}"
+    end
+  end # class JoinTablePair
+
+
+  class JoinTable
+    include Comparable
+
+    attr_reader   :name,     :datasource #, :str
+    attr_accessor :children, :depth
+
+    def initialize(name, datasource=nil)
+      @name       = name
+      @datasource = datasource
+      # @str        = nil
+      @children   = {}
+      @depth      = 0
+    end
+
+    def to_s
+      str = "[#{@datasource}].[#{@name}] :: (#{@depth}) :: #{@children.length} :: "
+      @children.each { |n,c| str << "#{n}, " }
+      return str
+    end
+
+    def <=>(anOther)
+      @str <=> anOther.str
+    end
+
+    def addChild child
+      # puts "#{@name}.addChild(#{child.name})"
+      # puts "children: #{@children}"
+      @children[child.name] = child if @children[child.name].nil?
+      # puts "children: #{@children}"
+    end
+
+    def child name
+      @children[name]
+    end
+
+  end # class JoinTable
+
+
+  class JoinTree
+
+    attr_reader  :datasource, :root, :maxdepth, :tables
+
+    def initialize  datasource
+      @datasource = datasource
+      @root       = nil
+      @maxdepth   = 0
+      @tables     = {}
+    end
+
+    def add host, dest
+      # puts "\nJT add() host: #{host}"
+      # puts "           dest: #{dest}"
+      from = @tables[host.name].nil? ? host : @tables[host.name]
+      to   = @tables[dest.name].nil? ? dest : @tables[dest.name]
+      from.addChild(to)
+      @tables[from.name] = from
+      @tables[to.name]   = to
+      if @root.nil?  || @root.name.eql?(to.name)
+        @root = from
+      end
+      setDepth(@root,1)
+    end
+
+    def table tableName
+      @tables[tableName]
+    end
+
+    def setDepth table, depth
+      # puts "-- setDepth(#{table.class}[#{table.name}] \t, #{depth})"
+      @tables[table.name].depth = depth
+      childrenDepth = depth+1
+      table.children.each { |n,c| setDepth(c,childrenDepth)}
+    end
+
+    def tableDepth tableName
+      @tables[tableName].nil? ? 0 : @tables[tableName].depth
+    end
+
+    def disp table
+      puts "%s %s %s  (%d)" % [' ' * table.depth, '-' * table.depth, table.name, table.depth]
+      table.children.each { |n,c| disp c}
+    end
+
+    def iterate
+      disp @root
+    end
+
+    def to_s
+      str = "root: #{@root}\ntables"
+      @tables.each do |t| 
+        str << " tbl:: #{t}"
+      end
+      return str
+    end
+
+  end # class JoinTree  
+
+end # module Twb