twb 1.0.5 → 1.9.1

data/lib/twb/fieldcalculation.rb

@@ -1,4 +1,4 @@
-#  Copyright (C) 2014, 2015  Chris Gerrard
+#  Copyright (C) 2014, 2015, 2017  Chris Gerrard
 #
 #  This program is free software: you can redistribute it and/or modify
 #  it under the terms of the GNU General Public License as published by
@@ -19,141 +19,144 @@ module Twb
 
   class FieldCalculation
 
-    attr_reader :node,    :fieldNode,       :dataSource
-    attr_reader :formula, :resolvedFormula, :formulaLines,  :formulaFlat
-    attr_reader :uiname,  :techname,        :caption
+    attr_reader :node,       :field,    :fieldNode,  :dataSource
+    attr_reader :uiname,     :techname, :caption
+    attr_reader :has_formula
+    attr_reader :formula         
+    attr_reader :formulaUC
+    attr_reader :formulaResolved
+    attr_reader :formulaFlat 
+    attr_reader :formulaFlatResolved
+    attr_reader :formulaLines 
+    attr_reader :is_tableCalc
+    attr_reader :is_lod,  :lodCodePos
     attr_reader :class,   :scopeIsolation
-    attr_reader :fields,  :resolvedFields,  :remoteFields
+    attr_reader :fields,  :remoteFields,  :calcFields
     attr_reader :comments
 
-    def initialize(calcNode, datasource=nil)
-      if calcNode
-        @node         = calcNode
-        @fieldNode    = @node.at_xpath('..')
+    @@tableCalcs = [ 'FIRST',        'INDEX', 'LAST', 'SIZE',
+                     'LOOKUP',       'PREVIOUS_VALUE',
+                     'RANK',         'RANK_DENSE',    'RANK_MODIFIED', 'RANK_PERCENTILE', 'RANK_UNIQUE',
+                     'RUNNING_AVG',  'RUNNING_COUNT', 'RUNNING_MAX',   'RUNNING_MIN',     'RUNNING_SUM',
+                     'SCRIPT_BOOL',  'SCRIPT_INT',    'SCRIPT_REAL',   'SCRIPT_STR',
+                     'TOTAL',
+                     'WINDOW_SUM',   'WINDOW_AVG',    'WINDOW_COUNT',  
+                     'WINDOW_MIN',   'WINDOW_MEDIAN', 'WINDOW_MAX',   'WINDOW_PERCENTILE',
+                     'WINDOW_STDEV', 'WINDOW_STDEVP',
+                     'WINDOW_CORR',  'WINDOW_COVAR',  'WINDOW_COVARP', 
+                     'WINDOW_VAR',   'WINDOW_VARP'
+                   ]
+
+    def initialize(calcField, datasource=nil)
+      raise ArgumentError.new("FieldCalculation must be initialized with a CalculatedField, has been provided with a #{calcField.class}") if calcField.class != Twb::CalculatedField
+      @field = calcField
+      calcNode = calcField.node
+      @istableCalc = false
+      # puts "FieldCalculation calcNode.nil? :: #{calcNode.nil?} "
+      unless calcNode.nil?
+        @node         = calcNode.at_xpath('./calculation')
+        @fieldNode    = calcField.node
         @dataSource   = datasource
         @class        = attribText(@node, 'class')
         @remoteFields = {}
         if 'categorical-bin'.eql? @class
-          # puts calcNode
           # <calculation class='categorical-bin' 
           #              column='[Calculation_507569757376950272]' 
           #              default='&quot;Other&quot;' 
           #              new-bin='true'>
-          @techname  = calcNode.attribute('column').text.gsub(/^\[|\]$/,'')   # assumes the column attribute exists
+          @techname  = @node.attribute('column').text.gsub(/^\[|\]$/,'')   # assumes the column attribute exists
           @caption   = attribText(@fieldNode, 'caption')
           uiname     = if datasource.nil?
                           @techName
                        else
                           datasource.fieldUIName(@techname)
                        end
-          @uiname    = "#{uiname} (group)"
-          @formula   = "grouped '#{uiname}' values"
-          @comments  = [ ]
-          @resolvedFields = [ {:field => uiname, :fieldui => uiname, :source => nil} ]
+          @uiname    = "#{uiname} <<group>>"
+          @has_formula  = true
+          @formula      = "grouped <<#{uiname}>> values"
+          @formulaLines = [ @formula ] 
+          @formulaFlat  = @formula 
+          @comments     = [ ]
+          @is_lod       = false
         else
-          # field names
-          # puts "\n****** "
-          # puts "   fnode: #{@fieldNode}\n****** "
-          @caption     = @fieldNode.attribute('caption').nil? ? nil : @fieldNode.attribute('caption').text
-          @techname    = @fieldNode.attribute('name').text.gsub(/^\[|\]$/,'')   # assumes the name attribute exists
-          @uiname      = @caption.nil? ? @techname : @caption
-          # puts " caption: #{@caption}"
-          # puts "techname: #{@techname}"
-          # puts "  uiname: #{@uiname}"
+          @caption   = calcField.caption
+          @techname  = calcField.name
+          @uiname    = calcField.uiname
           #--
           @scopeIsolation = attribText(@node, 'scope-isolation') 
           #-- Formula --
           @has_formula    = @node.has_attribute?('formula')
           if @has_formula
-            @formula      = @node.attribute('formula').text
+            @formula      = @node.attribute('formula').text.gsub(/\r\n/,"\n")
+            @formulaUC    = @formula.upcase
             @formulaLines = formula.split(/\n|\r\n/)
             @formulaFlat  = flattenFormula(@formulaLines)
             @comments     = getComments(@formulaLines)
-          elsif @class.eql? 'categorical-bin'
-            @formula      = '<<Group>>'
-            @formulaLines = [ @formula ] 
-            @formulaFlat  = [ @formula ] 
-            @comments     = [ ]
+            @lodCodePos   = @formula =~ /^[ ]*{[ ]*(fixed|include|exclude)[ ]*/i
+            @is_lod       = !lodCodePos.nil? && @lodCodePos >= 0
+            @is_tableCalc = @@tableCalcs.any? { |tc| @formulaUC.include?(tc) }  #assessTableCalc @formula
+            # puts "#{@lodCodePos} \t #{@is_lod} \t #{@is_lod.class} \t => #{@formula}"
           end
-          #-- Fields      --
-          parseFormFields # establishes @ fields
-          resolveFields   # establishes @ resolvedFields
         end
       end
     end
 
+    # def assessTableCalc formula
+    #   @@tableCalcs.any? { |tc| string.include?(tc) }
+    # end
+
     def attribText(node, attribute)
       node.attribute(attribute).nil? ? nil : node.attribute(attribute).text
     end
 
 
     def parseFormFields
-      @fields = Set.new []
-      formula = formulaFlat
+      @fields     = Set.new []
+      @calcFields = SortedSet.new []
+      formula = @formulaFlat
       if !formula.nil? && formula.include?('[') && formula.include?(']')
-          noSqLits =  formula.gsub(/'[\[\.\]]+'/, ' ')
+          noSqLits =  formula.gsub( /'[\[\.\]]+'/, ' ')
           flatForm = noSqLits.gsub( /\n/, ' ')
           stripFrt = flatForm.gsub( /^[^\[]*[\[]/    , '['      )
           stripBck = stripFrt.gsub( /\][^\]]+$/      , ']'      )
           stripMid = stripBck.gsub( /\][^\]]{2,}\[/  , ']]..[[' )
           stripCom = stripMid.gsub( /\][ ]*,[ ]*\[/  , ']]..[[' )
-          stripFns = stripMid.gsub( /\][ ]*[\*\/+\-,][ ]*\[/  , ']]..[[' )
+          stripFns = stripMid.gsub( /\][ ]*[\*\/+\-,=][ ]*\[/  , ']]..[[' )
           fields   = stripFns.split(']..[')
           fields.each { |field| @fields.add field.gsub(/^\[|\]$/, '')}
+          fields.each do |field| 
+            cf = CalculationField.new( field.gsub(/^\[|\]$/, ''), @dataSource, @dataSource.workbook )
+            @calcFields.add cf
+          end
       end
     end
 
-
-    def resolveFields
-      @resolvedFields = []
-      @fields.each do |field|
-        rawField = field.gsub(/^\[|\]$/,'')
-        parts    = rawField.split('].[')
-        if parts.length > 1
-          source    = parts[0]
-          fieldCalc = parts[1]
-          fieldui   = fieldCalc
-        else
-          source    = nil
-          fieldCalc = parts[0]
-          fieldui   = @dataSource.fieldUIName(fieldCalc)
-        end
-        hash = { :field => fieldCalc, :fieldui => fieldui, :source => source }
-        @resolvedFields << hash
-      end
-    end
-
-
-    def resolvedFormula
-      @resolvedFormula ||= resolveFormula
+    def formulaResolved
+      @formulaResolved ||= resolveFormula
     end
-
+    
     def resolveFormula
-      # puts "resolveFormula :\n-- formula --\n %s \n--     @dataSource.nil? : %s \n-- @resolvedFields.nil? : %s \n--" % [ @formula, @dataSource.nil?, @resolvedFields.nil? ]
       formula = @formula
-      unless @dataSource.nil? || @formula.nil? || @resolvedFields.nil?
-        @resolvedFields.each do |rf|
-          field  = rf[:field]
-          source = rf[:source]
-          # puts "  field: #{field}"
-          # puts "    src: #{source}"
-          if source.nil?
-            fieldInternal = field # "[#{rf[:field]}]"
-            fieldUI       = @dataSource.fieldUIName(field)
-            # puts "     f?: #{formula.include?(field)}"
-            # puts "    fui: #{fieldUI}"
-            formula = formula.gsub(field,fieldUI) unless fieldUI.nil?
-            # puts "formula:\n--\n#{formula}"
-          else
-            @remoteFields[source] = SortedSet.new unless @remoteFields.include? source
-            @remoteFields[source].add field
-          end
+      parseFormFields # - extracts the fields from the formula; as persisted they're the internal names
+     # puts "::++ #{formula}"
+      @calcFields.each do |calcField|
+        if calcField.techUIdiff
+         # puts ":::: #{calcField.techCode}  //  #{calcField.uiCode}"
+          formula = formula.gsub(calcField.techCode,calcField.uiCode)
+         # puts ":--: #{formula}"
         end
       end
-      # puts "resolvedFormula:\n==\n#{formula}\n--"
       return formula
     end
 
+    def formulaFlatResolved
+      @formulaFlatResolved ||= flattenResolvedFormula
+    end
+
+    def flattenResolvedFormula
+      formula = formulaResolved
+      formula.gsub(/\n/, '  ')
+    end
 
     def flattenFormula lines
       formula = ''
@@ -175,7 +178,80 @@ module Twb
       return comments.strip
     end
 
-
   end  # class FieldCalculation
 
+
+
+  class CalculationField
+    # is a field used in a calculation, resolved into it's human-meaningful form
+
+    include Comparable
+
+    attr_reader :techName,      :techCode     
+    attr_reader :uiName,        :uiCode 
+    attr_reader :dataSource,    :dataSourceRef, :dataSourceExists
+    attr_reader :fqName
+    attr_reader :techUIdiff
+
+    def initialize code, datasource, workbook
+     # puts "\nCalculationField <= \n\t  #{code}"
+      @dataSource       = datasource.uiname
+      @dataSourceRef    = :local
+      @dataSourceExists = true
+      @techUIdiff       = false
+      @uiName           = ''
+      rawCode  = code.gsub(/^\[|\]$/,'')
+      parts    = rawCode.split('].[')
+      # puts "\n\nField: #{code}  \t parts: #{parts.length} - #{parts.inspect}"
+      if parts.length == 1
+        @techName = parts[0]
+        @techCode = "[#{parts[0]}]"
+        if datasource.nil?
+          @uiName = @techName
+          @uiCode = @techCode
+          @techUIdiff = false
+        else # !datasource.nil?
+          # puts "#{@techName} \t:: #{datasource.fieldUIName(@techName).nil?} \t:: #{datasource.fieldUIName(@techName)} "
+          @uiName = datasource.fieldUIName(@techName).nil? ? @techName : datasource.fieldUIName(@techName) 
+          @uiCode = @uiName.nil? ? @techCode :  "[#{@uiName}]"
+          @techUIdiff = !@techCode.eql?(@uiCode)
+        end
+      else # parts.length <> 1
+        rdstech   = parts[0]
+        calcField = parts[1]
+        @uiName   = calcField
+        @dataSource    = rdstech
+        @dataSourceRef = :remote
+        @techCode = "[#{rdstech}].[#{calcField}]"
+        workbook  = datasource.workbook
+        remoteds  = workbook.nil? ? nil : workbook.datasource(rdstech)
+       # puts "\t  twb: #{workbook.class}  /  remoteds: #{remoteds.class}  : #{remoteds.nil? ? "<<NOT FOUND:#{rdstech}:>>" : remoteds.uiname} "
+        #--
+        if remoteds.nil? || remoteds.fieldUIName(calcField).nil?
+          @uiName     = calcField
+          @uiCode     = "[<<NOT FOUND>>#{rdstech}].[#{calcField}]"
+          @techUIdiff = true
+          @dataSourceExists = false
+        else !remoteds.nil?
+          @dataSource = remoteds.uiname
+          @uiName     = remoteds.fieldUIName(calcField)
+          @uiCode     = "[#{@dataSource}].[#{@uiName}]"
+          @techUIdiff = !@techCode.eql?(@uiCode)
+          @dataSourceExists = true
+        end
+      end
+      @fqName = "#{@dataSource}::#{@uiName}"
+    end # initialize
+
+    def <=>(other)
+      # myName    = @uiName.nil? ? '' : @uiName
+      # otherName = other.uiName.nil? ? "" : other.uiName
+      # # puts "#{@uiName} / #{myName} <=>  #{otherName} / #{other.uiName}"
+      # # puts "#{@uiName.nil?} // #{other.uiName.nil?}"
+      # myName <=> otherName
+      @fqName <=> other.fqName
+    end
+
+  end # class CalculationField
+
 end  # module Twb

data/lib/twb/localfield.rb

@@ -21,44 +21,47 @@ module Twb
 
   class LocalField
 
-    attr_reader :type, :node, :tableauname, :dbname, :datatype, :role, :type, :hidden, :caption, :aggregation, :uiname, :calculation, :comments
+    attr_reader :node, :type, :datatype, :name, :ordinal
 
     def initialize fieldNode
+      puts "lf:: #{fieldNode}"
       @node        = fieldNode
       @type        = 'local'
-      @tableauname = @node.attr('name')
-      @dbname      = @node.attr('name').gsub(/^\[/,'').gsub(/\]$/,'')
       @datatype    = @node.attr('datatype')
-      @role        = @node.attr('role')
-      @type        = @node.attr('type')
-      @hidden      = @node.attr('hidden')
-      @caption     = @node.attr('caption')
-      @aggregation = @node.attr('aggregation')
-      @calculation = getCalculation
-      @comments    = getComments
-      @uiname      = if @caption.nil? || @caption == '' then @dbname else @caption end
-      return self
+      @name        = @node.attr('name')
+      @ordinal     = @node.attr('ordinal')
+      # @dbname      = @node.attr('name').gsub(/^\[/,'').gsub(/\]$/,'')
+      # @datatype    = @node.attr('datatype')
+      # @role        = @node.attr('role')
+      # @type        = @node.attr('type')
+      # @hidden      = @node.attr('hidden')
+      # @caption     = @node.attr('caption')
+      # @aggregation = @node.attr('aggregation')
+      # @calculation = getCalculation
+      # @comments    = getComments
+      # @uiname      = if @caption.nil? || @caption == '' then @dbname else @caption end
+      # return self
     end
 
-    def getCalculation
-      calcNode = @node.at_xpath("./calculation")
-      FieldCalculation.new(calcNode) unless calcNode.nil?
-    end
+    # def getCalculation
+    #   calcNode = @node.at_xpath("./calculation")
+    #   FieldCalculation.new(calcNode) unless calcNode.nil?
+    # end
 
-    def getComments
-      comments = ''
-      runs = node.xpath('./desc/formatted-text/run')
-      runs.each do |run|
-        unless run.nil?
-          comments += run.text
-        end
-      end
-      return comments
-    end
+    # def getComments
+    #   comments = ''
+    #   runs = node.xpath('./desc/formatted-text/run')
+    #   runs.each do |run|
+    #     unless run.nil?
+    #       comments += run.text
+    #     end
+    #   end
+    #   return comments
+    # end
 
-    def remove_attribute attribute
-      @node.remove_attribute(attribute)
-    end
+    # def remove_attribute attribute
+    #   @node.remove_attribute(attribute)
+    # end
 
   end
 

data/lib/twb/metadatafield.rb

@@ -17,37 +17,79 @@ require 'nokogiri'
 
 module Twb
 
-  # Assumption: A field can only be either a MetadataField or a LocalField, not both in a given Workbook data connection.
-
   class MetadataField
 
-    attr_reader :node, :aggregation, :containsnull, :localname, :localtype, :ordinal
-    attr_reader :parentname, :precision, :remotealias, :remotename, :remotetype, :width, :name
+    include Comparable
+
+    attr_reader     :node,         :class
+    attr_accessor   :source
+    attr_reader     :parentName,   :table        # parentName is the Db table
+    attr_reader     :name,         :caption,     :family
+    attr_reader     :localName,    :localType
+    attr_reader     :remoteName,   :remoteAlias  
+    attr_reader     :remoteType,   :aggregation 
+    attr_reader     :containsNull, :ordinal
+    attr_reader     :precision,    :width,       :scale
+
+    # Child nodes of <metadata-record nodes 
+    # <aggregation
+    # <approx-count
+    # <attributes
+    # <caption
+    # <collation
+    # <contains-null
+    # <family
+    # <layered
+    # <local-name
+    # <local-type
+    # <ordinal
+    # <parent-name
+    # <precision
+    # <remote-alias
+    # <remote-name
+    # <remote-type
+    # <scale
+    # <statistics
+    # <width
+
 
     def initialize fieldNode
+      @class             = fieldNode.attribute('class').text
       @node              = fieldNode
+      @parentName        = load 'parent-name'
+      @family            = load 'family'
+      @table             = @family.nil? ? @parentName : @family
+      @remoteName        = load 'remote-name' 
+      @remoteAlias       = load 'remote-alias'
+      @remoteType        = load 'remote-type'
+      @caption           = load 'caption'   
+      @localName         = load 'local-name'
+      @name              = @caption.nil? ? @localName : @caption
       @aggregation       = load 'aggregation'
-      @containsnull      = load 'contains-null'
-      @localname         = load 'local-name'
-      @localtype         = load 'local-type'
+      @containsNull      = load 'contains-null'
+      @localType         = load 'local-type'
       @ordinal           = load 'ordinal'
-      @parentname        = load 'parent-name'
       @precision         = load 'precision'
-      @remotealias       = load 'remote-alias'
-      @remotename        = load 'remote-name'
-      @name              = @remotename
-      @remotetype        = load 'remote-type'
+      @scale             = load 'scale'
       @width             = load 'width'
-      return self
+      @id                = "'%s::%s' " % [@table,@remoteName]   
+      # return self
     end
 
     def load nodeName
       node = @node.at_xpath(nodeName)
-      val  = if node.nil? then  node else node.text.strip end
-      # puts "==== MD node:'#{nodeName}' \t nil?'#{node.nil?}' \t == val:#{val} \t = '#{node}' "
+      val = node.nil? ? nil : node.text.strip.gsub(/^\[|\]$/,'')
       return val
     end
 
+    def to_s
+      @id
+    end
+
+    def <=>(other)
+      @id <=> other.id
+    end
+
   end
 
 end