twb 2.2.1 → 3.7.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 0ac4493d38cdcff16c2d5cb96873ba8356e80271
-  data.tar.gz: 8f77ffab827d962ec69aae1061d72978a65111ed
+  metadata.gz: fd921c477a3ab3ef1de872935a8057bd4fd5ea27
+  data.tar.gz: de2bc297a944a544bb0f251ac5cc9f972e1f763c
 SHA512:
-  metadata.gz: 0f146956037bfdbd37161ee9a631dc86827afa4af34470199beb2553b152cdeb95201994b088a2028ac89cb5d54e728ff70af4c794e1f9d7a1a903d3af9ab5a2
-  data.tar.gz: ef4a7e6e3435b0e3bfa4d9ba096f8f055ec40ffc865196f0923b04cbd9818a9047c62799fb0f505ae1d940269719c71446e0227c44d79f15285fef058770bed8
+  metadata.gz: 46becdab0ca9837b76ddd7ffc46e26433ab130bda120c5150ddce3575da04e80309874dfef9ca69531556b6185b46131889b62fe3f0a05e58b341be83ef04d56
+  data.tar.gz: 41f15a08b3b9971c4ca237ea5a166c212488c6e13a5ee3ca59613b8894d7b23d619e9d54fbc3c41280b4fc543c9109af1985befb92a7b42dc2bea00dc14f865a

data/lib/twb.rb

@@ -13,6 +13,8 @@
 #  You should have received a copy of the GNU General Public License
 #  along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
+require_relative 'twb/tabclass'
+require_relative 'twb/tabtool'
 require_relative 'twb/dashboard'
 require_relative 'twb/datasource'
 require_relative 'twb/docdashboard'
@@ -26,6 +28,7 @@ require_relative 'twb/worksheet'
 require_relative 'twb/action'
 require_relative 'twb/columnfield.rb'
 require_relative 'twb/calculatedfield'
+require_relative 'twb/codedfield'
 require_relative 'twb/fieldcalculation'
 require_relative 'twb/docdashboardimagevert'
 require_relative 'twb/docdashboardwebvert'
@@ -33,9 +36,15 @@ require_relative 'twb/util/twbDashSheetDataDotBuilder'
 require_relative 'twb/util/dotFileRenderer'
 require_relative 'twb/util/htmllistcollapsible'
 require_relative 'twb/util/xraydashboards'
+require_relative 'twb/util/graph'
 require_relative 'twb/util/graphnode'
 require_relative 'twb/util/graphedge'
 require_relative 'twb/util/graphedges'
+require_relative 'twb/util/gml'
+require_relative 'twb/util/cypher'
+require_relative 'twb/util/cypherpython'
+require_relative 'twb/util/fielddomainloader'
+require_relative 'twb/util/docprep'
 require_relative 'twb/analysis/documentedfieldsmarkdownemitter'
 require_relative 'twb/analysis/annotatedfieldsCSVEmitter'
 require_relative 'twb/analysis/calculatedfields/calculatedfieldsanalyzer'
@@ -43,10 +52,13 @@ require_relative 'twb/analysis/calculatedfields/markdownemitter'
 require_relative 'twb/analysis/calculatedfields/csvemitter'
 require_relative 'twb/analysis/datasources/DataSourceFieldsCSVEmitter'
 require_relative 'twb/analysis/datasources/DataSourceTableFieldsCSVEmitter'
+require_relative 'twb/analysis/datasources/googlesheetdatasourcesanalyzer'
 require_relative 'twb/analysis/Sheets/WorksheetDataStructureCSVEmitter'
+require_relative 'twb/analysis/Sheets/sheetfiltersanalyzer'
+require_relative 'twb/analysis/Sheets/sheetfieldsanalyzer'
 
 # Represents Tableau Workbooks and their contents.
 #
 module Twb
-  VERSION = '2.2.1'
+  VERSION = '3.7.2'
 end

data/lib/twb/action.rb

@@ -18,7 +18,7 @@ require 'digest/md5'
 
 module Twb
 
-  class Action
+  class Action < TabClass
 
     # Notes
     # Sources:
@@ -68,6 +68,10 @@ module Twb
       return self
     end
 
+    def id
+        @id ||= @id = @name.hash
+    end
+
     def process
 
     end

data/lib/twb/analysis/AnnotatedFieldsCSVEmitter.rb

@@ -37,6 +37,7 @@ module Analysis
     @@csvFileName = @@csvFileType + '.csv'
 
     def initialize
+      # puts "AnnotatedFieldsCSVEmitter - initializing"
       @csvFile     = CSV.open(@@csvFileName,'w')
       @csvFile     << @@csvHeader
       @csvRecords  = Set.new
@@ -63,6 +64,7 @@ module Analysis
       @twb         = twb                    if twb.instance_of? Twb::Workbook
       @twb         = Twb::Workbook.new(twb) if twb.instance_of? String
       raise ArgumentError.new("ERROR in Workbok processing: '#{twb}' must be a Workbook (class) or the name of a Workbook (String), is a #{twb.class} \n ") unless @twb.is_a? Twb::Workbook
+      # puts "Processing #{@twb.name}"
       # --
       dss = @twb.datasources
       dss.each do |ds|
@@ -78,6 +80,7 @@ module Analysis
     end # def processTwb twb
 
     def recordField field
+      # puts "- #{field.uiname} :: #{field} ::: #{field.node}"
       @fieldNum += 1
       @lineNum   = 0
       field.comment.each do |line|

data/lib/twb/analysis/CalculatedFields/CalculatedFieldsAnalyzer.rb

@@ -1,4 +1,4 @@
-#  calculatedfieldsanalyzer.rb - this Ruby script Copyright 2017 Christopher Gerrard
+#  calculatedfieldsanalyzer.rb - this Ruby script Copyright 2017, 2018 Christopher Gerrard
 #
 #  This program is free software: you can redistribute it and/or modify
 #  it under the terms of the GNU General Public License as published by
@@ -17,14 +17,16 @@ require 'nokogiri'
 require 'twb'
 require 'set'
 require 'csv'
-require 'logger'
 
 module Twb
 module Analysis
 
-  class CalculatedFieldsAnalyzer
+  class CalculatedFieldsAnalyzer < Twb::Util::Graph
 
-      attr_reader :calculatedFieldsCount, :formulaFieldsCount, :dataFiles
+      include TabTool
+
+      attr_reader   :calculatedFieldsCount, :formulaFieldsCount, :dataFiles
+      attr_accessor :ttdocdir
 
       @@ttlogfile     = 'CalculatedFieldsAnalyzer.ttlog'
       @@gvDotLocation = 'C:\\tech\\graphviz\\Graphviz2.38\\bin\\dot.exe'
@@ -40,12 +42,20 @@ module Analysis
                                    'Class',
                                    'Scope Isolation',
                                    'Formula Length',
-                                   'Formula Code',
                                    'Formula',
+                                   'Formula (tech)',
                                    'Formula Comments',
                                    'Formula LOD?'
                                   ]
 
+      @@calcLinesCSVFileName    = 'TwbCalculatedFieldFormulaLines.csv'
+      @@calcLinesCSVFileHeader  = ['Calc Field #',
+                                   'Workbook',      'Workbook Dir',
+                                   'Data Source',   'Data Source Caption', 'Data Source Name (tech)',
+                                   'Field Name',    'Field Caption',       'Field Name (tech)',
+                                   'Formula',       'Formula Line #',      'Formula Line'
+                                  ]
+
       @@formFieldsCSVFileName   = 'TwbFormulaFields.csv'
       @@formFieldsCSVFileHeader = ['Rec #',
                                    'Workbook',   'Workbook Dir',
@@ -71,258 +81,234 @@ module Analysis
 DOTHEADER
 
       def initialize
+          #-- Logging setup --
+          @ttdocdir = 'ttdoc'
+          # logfile   = docFile(@@ttlogfile)
+          # @logger   = Logger.new(logfile)
+          # @logger.level = Logger::DEBUG
+          emit '  Initializing CalculatedFieldsAnalyzer'
+          #-- CSV records collectors
           @csvCalculatedFields = []
           @csvFormulaFields    = []
-          # @testFile = File.open('testCSV.csv','w')
-          # @testFile.puts @@calcFieldsCSVFileHeader.inspect
-          #-- Logging setup --
-          @logger = Logger.new(@@ttlogfile)
-          @logger.level = Logger::DEBUG
+          @csvFormulaLines     = []
           #-- Counters setup --
-            @twbCount              = 0
-            @calculatedFieldsCount = 0
-            @formulaFieldsCount    = 0
-          # --
-            @referencedFields     = SortedSet.new
-          # --
-            @dataFiles   = {'TwbCalculatedFields.csv' => 'Calculated Fields & their Formulas', 'TwbFormulaFields.csv' => 'Fields referenced in Formulas'}
-          # --
-            @localEmit  = false
-            emit  "\n\nLogging activity to: #{File.basename(@@ttlogfile)}"
-            @imageFiles  = []
+          @twbCount              = 0
+          @calculatedFieldsCount = 0
+          @formulaFieldsCount    = 0
+          #--
+          @referencedFields     = SortedSet.new
+          #--
+          emit "init CSV #{@@calcFieldsCSVFileName}"
+          @csvCF = File.open(docFile(@@calcFieldsCSVFileName), 'w')
+          @csvCF.puts @@calcFieldsCSVFileHeader.to_csv
+          # @csvCF.close
+          #--
+          emit "init CSV #{@@calcLinesCSVFileName}"
+          @csvCFLs = File.open(docFile(@@calcLinesCSVFileName), 'w')
+          @csvCFLs.puts @@calcLinesCSVFileHeader.to_csv
+          # @csvCFLs.close
+          #--
+          emit "init CSV #{@@formFieldsCSVFileName}"
+          @csvFF = File.open(docFile(@@formFieldsCSVFileName), 'w')
+          @csvFF.puts @@formFieldsCSVFileHeader.to_csv
+          # @csvFF.close
+          #--
+          @dataFiles   = { @@calcFieldsCSVFileName => 'Calculated Fields & their Formulas', 
+                           @@calcLinesCSVFileName  => 'Calculated Fields & individual Formula Lines',
+                           @@formFieldsCSVFileName => 'Fields referenced in Formulas'
+                         }
+          #--
+          @localEmit  = false
+          @imageFiles  = []
+      end
+
+      def initDocDir
+        return if $ttdocdir.nil?
+        return if ''.eql? $ttdocdir
+        return if Dir.exists?($ttdocdir)
+        if File.exists? $ttdocdir
+            $ttdocdir = nil
+            return
+        end
+        Dir.mkdir $ttdocdir
       end
 
-      def processTWB twbWithDir
-        twb = File.basename(twbWithDir)
-        @twb = Twb::Workbook.new twbWithDir
-        emit  "- Workbook: #{twbWithDir}"
+      def docFile name
+        @ttdocdir.nil? ? name : "#{@ttdocdir}\\#{name}"
+      end
+
+      def processTWB workbook
+        @twb = workbook.is_a?(String) ? Twb::Workbook.new(workbook) : workbook
+        throw Exception unless @twb.is_a? Twb::Workbook
+        emit  "- Workbook: #{workbook}"
         emit  "   version: #{@twb.version}"
-        return if twbWithDir.end_with? == "Tableau Calculated Fields Analyses.twb"
-        twbDir   = File.dirname(File.expand_path(twbWithDir))
-        edges    = Set.new
-        # -- processing
+        @twbDir   = @twb.dir #File.dirname(File.expand_path(workbook))
+        @edges    = Set.new
+        #-- processing
         dss = @twb.datasources
-        puts "    # data sources: #{dss.length}"
-        twbRootFields = Set.new
+        # puts "    # data sources: #{dss.length}"
+        @twbRootFields = Set.new
         @twbFields = {}
+        @nodes     = Set.new
         dss.each do |ds|
-          puts "\t\t - #{ds.uiname}  \t\t #{ds.calculatedFields.length}"
-          next if ds.Parameters?  # don't process the Parameters data source
-          ds.calculatedFields.each do |calcField|
-            emit "HANDLING CALCULATED FIELD::  #{calcField}"
-            emit "                         ::  #{calcField.calculation.formula}"
-            emit "                         ::  #{calcField.calculation.formulaResolved}"
-            dsTechName  = ds.name
-            dsCaption   = ds.caption
-            dsName      = ds.uiname
-            dsID        = dsTechName + ':::' + dsName
-            emit  "\n\n "
-            emit  "======================================================"
-            emit  "======================================================"
-            emit   "=======    DATA SOURCE: #{ds.uiname}    ====== "
-            emit  "======================================================"
-            emit  "======================================================\n\n "
-            dsGraphNode = Twb::Util::Graphnode.new(name: dsName, id: dsID, type: :TwbDataConnection, properties: {workbook: twbWithDir})
-            emit  "\t      dsgnode: #{dsGraphNode}"
-            fieldUINames      = ds.fieldUINames
-            emit  "ds.calculatedFields :: nil? #{ds.calculatedFields.nil?}"
-          end
+          # puts "\t\t - #{ds.uiname}  \t\t #{ds.calculatedFields.length}"
+          next if ds.Parameters?  # don't process the Parameters data source - Parameters' fields aren't Calculated fields for our purposes
+          # dataSourceNode = Twb::Util::Graphnode.new(name: ds.uiname, id: ds.id, type: ds, properties: {workbook: workbook})
+          # @nodes.add dataSourceNode
+          # ds.calculatedFields.each do |calcField|
+          # end
           processDataSource ds
         end
+        mapTwb
+        emitGml
+        @twbCount += 1
+      end
+
+      def emitGml
+        gml = Twb::Util::GML.new
+        gml.fileName = @twb.name
+        gml.nodes = @nodes
+        gml.edges = @edges
+        gml.render
       end
 
       def processDataSource ds
-        emit  "Datasource: '#{ds.uiname}'    -> #{ds.Parameters?}"
-        dsFields = {}
-        @twbFields[ds.uiname] = dsFields
-        # next if ds.Parameters?  # don't process the Parameters data source
-                                # it requires special handling, has different XML structure
-        #-- For tracking unreferenced (root) calculated fields = calculatedFields - referencedFields
-        calculatedFields = SortedSet.new
-        referencedFields = SortedSet.new
-        #--
-        dsTechName  = ds.name
-        dsCaption   = ds.caption
-        dsName      = ds.uiname
-        dsID        = dsTechName + ':::' + dsName
-        emit  "\n\n "
-        emit  "======================================================"
-        emit  "======================================================"
         emit   "=======    DATA SOURCE: #{ds.uiname}    ====== "
-        emit  "======================================================"
-        emit  "======================================================\n\n "
-        dsGraphNode = Twb::Util::Graphnode.new(name: dsName, id: dsID, type: :TwbDataConnection, properties: {workbook: twbWithDir})
-        emit  "\t      dsgnode: #{dsGraphNode}"
-        fieldUINames      = ds.fieldUINames
-        emit  "ds.calculatedFields :: nil? #{ds.calculatedFields.nil?}"
+        dsNodes               = Set.new
+        dsEdges               = Set.new
+        dsFields              = {}
+        @twbFields[ds.uiname] = dsFields
+        calculatedFields      = SortedSet.new
+        fieldFormulaLines     = [] 
+        referencedFields      = SortedSet.new
+        dataSourceNode = Twb::Util::Graphnode.new(name: ds.uiname, id: ds.id, type: ds, properties: {workbook: @twb.name})
+        @nodes.add dataSourceNode
+        #-- process Calculatred Fields
         ds.calculatedFields.each do |calcField|
-          emit "HANDLING CALCULATED FIELD::  #{calcField}"
-          emit  '--'
-          fldCaption,     = calcField.caption
-          fldTechName     = calcField.name
-          fldName         = calcField.uiname
-          dataType        = calcField.datatype 
-          role            = calcField.role     
-          type            = calcField.type     
-          fieldID         = fldTechName+'::'+dsName
-          calculatedFields.add fieldID
-          #--
-          dsFields[fldName] = calcField
-          srcGraphNode    = Twb::Util::Graphnode.new(name: fldName,     id: fieldID,      type: :CalculatedField, properties: {:DataSource => dsName})
-          dsFieldEdge     = Twb::Util::Graphedge.new(from: dsGraphNode, to: srcGraphNode, relationship: 'contains')
-          #--
-          emit  "\t     srfnode: #{srcGraphNode} "
-          emit  "\t     dsfedge: #{dsFieldEdge} "
-          edges.add dsFieldEdge
+          calculatedFields.add calcField.id
+          dsFields[calcField.uiname] = calcField
+          calcFieldNode    = Twb::Util::Graphnode.new(name: calcField.uiname, id: calcField.id, type: calcField, properties: {:DataSource => ds.uiname})
+          @nodes.add calcFieldNode
+          dsFieldEdge     = Twb::Util::Graphedge.new(from: dataSourceNode, to: calcFieldNode, relationship: 'contains')
+          @edges.add dsFieldEdge
           calculation     = calcField.calculation
-          emit "calculation: f? %8s -> %s " % [calculation.has_formula, calculation.formula ]
           if calculation.has_formula
-              formulaFlat     = calculation.formulaFlat
-              uiFormula       = formulaFlat.gsub(' XX ',' ')
-              formulaLOD      = formulaFlat.upcase =~ /^[ ]*{[ ]*(INCLUDE|FIXED|EXCLUDE)/
-              resolvedFields  = calculation.resolvedFields
-              resolvedFields.each do |rf|
-                emit  "\tRESOLVED FLD: #{rf.inspect}"
-                calcFieldName   = rf[:field]
-                if rf[:source].nil?
-                  calcFieldRef = "[%s]" % [ calcFieldName ]
-                  dispFieldRef = "[%s]" % [ ds.fieldUIName(calcFieldName) ]
-                else
-                  remoteDS     = @twb.datasource(rf[:source])
-                  if remoteDS.nil?
-                    calcFieldRef = "[DS_NOT_FOUND].[%s]" % [ calcFieldName ]
-                    dispFieldRef = calcFieldRef
-                  else
-                    remoteDSName = remoteDS.uiname 
-                    remoteDSFld  = remoteDS.fieldUIName(calcFieldName)
-                    calcFieldRef = "[%s].[%s]" % [ rf[:source],  calcFieldName ]
-                    dispFieldRef = "[%s].[%s]" % [ remoteDSName, remoteDSFld   ]
-                  end # remoteDS.nil?
-                end # rf[:source].nil?
-                emit  "\tcalcFieldRef: #{calcFieldRef}"
-                emit  "\tdispFieldRef: #{dispFieldRef}"
-                uiFormula    = uiFormula.gsub(calcFieldRef, dispFieldRef)
-              end # resolvedFields.each
+              #-- collect field formulas as single lines
               @csvCalculatedFields.push [
                                           @calculatedFieldsCount += 1,
-                                          twb,           
-                                          twbDir,
-                                          dsName,
-                                          dsCaption,
-                                          dsTechName,
-                                          fldName,
-                                          fldCaption,
-                                          fldTechName,
-                                          dsTechName + '::' + fldTechName,
-                                          dataType,
-                                          role,
-                                          type,
+                                          @twb.name,           
+                                          @twbDir,
+                                          ds.uiname,
+                                          ds.caption,
+                                          ds.name,
+                                          calcField.uiname,
+                                          calcField.caption,
+                                          calcField.name,
+                                          ds.name + '::' + calcField.name,
+                                          calcField.datatype,
+                                          calcField.role,
+                                          calcField.type,
                                           calculation.class,     
                                           calculation.scopeIsolation,
                                           calculation.formulaFlat.length, 
-                                          calculation.formulaFlat,    
-                                          uiFormula[0...200],
+                                          calculation.formulaResolved,    
+                                          calculation.formulaFlat,
                                           calculation.comments,
-                                          !formulaLOD.nil?
+                                          calculation.is_lod
                                         ]
-              resolvedFields.each do |rf|
-                emit  "\t\t        res field : #{rf[:field]}    "
-                emit  "\t\t        res source: #{rf[:source]}"
-                calcFieldName   = rf[:field]
-                calcDataSource  = rf[:source]
-                localDataSource = rf[:source].nil?   # if there isn't a rf[:source] value
-                                                       # the field is from this data source
-                                                       # else the field is from an alien data source (in the same workbook)
-                refDataSource     = localDataSource ? ds : @twb.datasource(calcDataSource)  # data source may not be in Workbook
-                refDataSourceName = !refDataSource.nil? ? refDataSource.uiname : calcDataSource + '\n***DATA CONNECTION NOT IN WORKBOOK***'
-                if !refDataSource.nil?
-                  dispFieldName   = refDataSource.fieldUIName(calcFieldName)
-                  calcFieldTable  = refDataSource.fieldTable(calcFieldName)
-                else
-                  dispFieldName   = calcFieldName
-                  calcFieldTable  = calcFieldName
-                end
-                emit  "\t\t       calc field : #{dispFieldName}          nil?<#{dispFieldName.nil?}>"
-                emit  "\t\t       data source: #{refDataSourceName}"
-                emit  "\t\t             table: #{calcFieldTable}         nil?<#{calcFieldTable.nil?}>"
-                properties = {'DataSource' => dsName, 'DataSourceReference' => 'local'}
-                if dispFieldName.nil?
-                   dispFieldName = "<#{calcFieldName}>::<#{calcDataSource}> UNDEFINED"
-                   properties['status'] = 'UNDEFINED'
-                end
-                calcFieldID = "#{calcFieldName}::#{refDataSourceName}"
-                if !localDataSource
-                  calcFieldID = "#{calcFieldName}:LDS:#{ds.uiname}:RDS:#{refDataSourceName}"
-                  properties['DataSourceReference'] = 'remote'
-                end
-                calcFieldTable = ds.fieldTable(calcFieldName)
-                calcFieldType  = calcFieldTable.nil? ? :CalculatedField : :DatabaseField
-                calcFieldNode  = Twb::Util::Graphnode.new(name: dispFieldName, id: calcFieldID,   type: calcFieldType, properties: properties)
-                fieldFieldEdge = Twb::Util::Graphedge.new(from: srcGraphNode,  to: calcFieldNode, relationship: 'references')
-                edges.add fieldFieldEdge
-                referencedFields.add calcFieldID
-                emit  "\t\t    calcFieldNode: #{calcFieldNode}"
-                emit  "\t\t        graphEdge: #{fieldFieldEdge}"
-                fldToDsNode = calcFieldNode
-                if !calcFieldTable.nil?
-                   tableID        = calcFieldTable + ':::' + ds.uiname
-                   tableName      = "-[#{calcFieldTable}]-"
-                   tableNode      = Twb::Util::Graphnode.new(name: tableName,      id: tableID,   type: :DBTable, properties: properties)
-                   fieldFieldEdge = Twb::Util::Graphedge.new(from: calcFieldNode,  to: tableNode, relationship: 'is a field in')
-                   edges.add fieldFieldEdge
-                   fldToDsNode    = tableNode
+              #-- collect individual formula lines
+              flnum = 0
+              emit "@@ FL: #{calcField.uiname}"
+              calculation.formulaResolvedLines.each do |fl|
+                emit "@@ FL:   => '#{fl}'"
+                fieldFormulaLines.push [ @calculatedFieldsCount,        # 'Calc Field #',
+                                         @twb.name,                     # 'Workbook',      
+                                         @twbDir,                       # 'Workbook Dir',
+                                         ds.uiname,                     # 'Data Source',   
+                                         ds.caption,                    # 'Data Source Caption', 
+                                         ds.name,                       # 'Data Source Name (tech)',
+                                         calcField.uiname,              # 'Field Name',    
+                                         calcField.caption,             # 'Field Caption',       
+                                         calcField.name,                # 'Field Name (tech)',
+                                         calcField.calculation.formulaFlatResolved, # 'Formula'
+                                         flnum += 1,                    # 'Formula Line #',
+                                         fl.start_with?(" ") ? "'#{fl}" : fl # 'Formula Line' - THIS IS A STUPID HACK NEEDED BECAUSE TABLEAU STRIPS LEADING BLANKS FROM CSV VALUES
+                                       ]
+              end
+              #-- collect fields referenced in formula
+              calculation.calcFields.each do |rf|
+                emit "  rf.name  :'#{rf.name}'"
+                emit "  rf.uiname:'#{rf.uiname}'"
+                properties = {'DataSource' => ds.uiname, 'DataSourceReference' => 'local', :source => rf}
+                refFieldNode  = Twb::Util::Graphnode.new(name: rf.uiname, id: rf.id,   type: rf.type, properties: properties)
+                @nodes.add refFieldNode
+                fieldFieldEdge = Twb::Util::Graphedge.new(from: calcFieldNode,  to: refFieldNode, relationship: 'references')
+                @edges.add fieldFieldEdge
+                referencedFields.add rf.id
+                refFieldTable = ds.fieldTable(rf.name)
+                emit "refFieldTable.nil? : #{refFieldTable.nil?}"
+                unless refFieldTable.nil?
+                  tableID        = refFieldTable + ':::' + ds.uiname
+                  tableName      = "||#{refFieldTable}||"
+                  tableNode      = Twb::Util::Graphnode.new(name: tableName,      id: tableID,   type: :DBTable, properties: properties)
+                  @nodes.add tableNode
+                  fieldFieldEdge = Twb::Util::Graphedge.new(from: refFieldNode,  to: tableNode, relationship: 'is a field in')
+                  @edges.add fieldFieldEdge
+                  # fldToDsNode    = tableNode
                 end
-                if !localDataSource
-                  alienDSNode = Twb::Util::Graphnode.new(       name: '==>' + refDataSourceName,
-                                                                  id: "#{ds.uiname}::::=>#{refDataSourceName}",
-                                                                type: :DBTable,
-                                                          properties: {'Home Source' => dsName, 'Remote Source' => refDataSourceName}
-                                                        )
-                  fieldFieldEdge  = Twb::Util::Graphedge.new(from: fldToDsNode,  to: alienDSNode, relationship: 'In Remote Data Source')
-                  edges.add fieldFieldEdge
-                end
-
-                # @csvFormulaFields.push [ @formulaFieldsCount+=1,
-
-
+                @csvFormulaFields.push [ 
+                                          @formulaFieldsCount += 1,
+                                          @twb.name,           
+                                          @twbDir,
+                                          ds.uiname,
+                                          calcField.uiname,
+                                          calculation.formulaFlat,
+                                          calculation.formulaResolved,
+                                          rf.name,
+                                          rf.uiname,
+                                          rf.id,
+                                          refFieldTable
+                                       ]
               end # resolvedFields.each do
           end # if calculation.has_formula
         end # ds.calculatedFields.each 
 
         dsRootFields = calculatedFields - referencedFields
         @referencedFields.merge referencedFields
-        #--
-        emit "--\nCalculated Fields\n-----------------"
-        calculatedFields.each { |f| emit f }
-        emit "--\nReferenced Fields\n-----------------"
-        referencedFields.each { |f| emit f }
-        emit "--\nDS Root Fields\n-----------------"
-        dsRootFields.each { |f| emit f }
-        emit "--"
-        # --
-        twbRootFields.merge dsRootFields
-        @twbCount += 1
-        mapTwb     twb, edges, twbRootFields
-        graphEdges twb, edges
+        @twbRootFields.merge dsRootFields
+        cypher     @twb.name
+        cypherPy   @twb.name
         emit "#######################"
-        #--
-        csvCF = File.open(@@calcFieldsCSVFileName, 'w')
-        csvCF.puts @@calcFieldsCSVFileHeader.to_csv
+        #-- record calculated fields 
+            # @csvCF = File.open(docFile(@@calcFieldsCSVFileName), 'a')
+            # @csvCF.puts @@calcFieldsCSVFileHeader.to_csv
+        emit "@@ record calculated fields  ds: #{ds.uiname}"
+        # @csvCF.open('a')
         @csvCalculatedFields.each do |r| 
-          csvCF.puts r.to_csv
+          @csvCF.puts r.to_csv
         end
-        csvCF.close
-        #--
-        csvFF = File.open(@@formFieldsCSVFileName, 'w')
-        csvFF.puts @@formFieldsCSVFileHeader.to_csv
+        #-- record individual formula lines 
+          # @csvCFLs = File.open(docFile(@@calcLinesCSVFileName), 'a')
+          # @csvCFLs.puts @@calcLinesCSVFileHeader.to_csv
+        emit "@@ individual formula lines  ds: #{ds.uiname}"
+        fieldFormulaLines.each do |ffl|
+          emit "@@@@ FFLA: #{ffl}" 
+          emit "@@@@ FFLB: #{ffl.to_csv}" 
+          emit "@@@@ FFLc: " 
+          @csvCFLs.puts ffl.to_csv
+        end
+        #-- record formula-referenced fields 
+          # @csvFF = File.open(docFile(@@formFieldsCSVFileName), 'a')
+          # @csvFF.puts @@formFieldsCSVFileHeader.to_csv
+        emit "@@ formula-referenced fields ds: #{ds.uiname}"
         @csvFormulaFields.each do |r| 
-          csvFF.puts r.to_csv
+          @csvFF.puts r.to_csv
         end
-        csvFF.close
         #--
         return @imageFiles
       end # def processDataSource
 
+
       def emitCalcfield calcField
         emit "\t FIELD    cap :: #{calcField.caption} "
         emit "\t         tname:: #{calcField.name}"
@@ -330,9 +316,12 @@ DOTHEADER
         emit "\t       formula:: #{calculation.formulaFlat}"
       end
 
-      def mapTwb twb, edges, rootFields
-        dotFile     = initDot twb
-        dotFileName = File.basename dotFile
+      def mapTwb 
+        twb        = @twb.name
+        rootFields = @twbRootFields
+        dotStuff    = initDot twb
+        dotFile     = dotStuff[:file]
+        dotFileName = dotStuff[:name]
         dotFile.puts "\n //  subgraph cluster_1 {"
         dotFile.puts "   //    color= grey;"
         dotFile.puts ""
@@ -340,85 +329,94 @@ DOTHEADER
         # this two step process coalesces the edges into a unique set, avoiding duplicating the dot
         # file entries, and can be shrunk when graph edges expose the bits necessary for management by Set
         emit "\n========================\nLoading Edges\n========================\n  From DC?  Referenced?  Edge \n  %s  %s  %s" % ['--------', '-----------', '-'*45]
-        edges.each do |e|
+        @edges.each do |e|
           # don't want to emit edge which is from a Data Connection to a
           # Calculated Field which is also referenced by another calculated field
           isFromDC   = e.from.type == :TwbDataConnection
           isRefField = @referencedFields.include?(e.to.id)
           edgesAsStrings.add(e.dot) unless isFromDC && isRefField
+          # emit "   ES        #{e.dot}"
+          # emit "   ES  from  #{e.from}"
+          # emit "   ES    to  #{e.to}"
         end
         emit "------------------------\n "
         edgesAsStrings.each do |es|
           dotFile.puts "        #{es}"
-          emit "           #{es}"
         end
         emit "========================\n "
         dotFile.puts ""
         dotFile.puts "   // }"
-        dotFile.puts "\n\n  //  4--------------------------------------------------------------------"
-        # "table::JIRA_HARVEST_Correspondence__c::Jira" [label="JIRA_HARVEST_Correspondence__c"]
-        nodes = SortedSet.new
-        edges.each do |e|
-          nodes.add e.from.dotLabel
-          nodes.add e.to.dotLabel
-        end
-        nodes.each do |n|
-          dotFile.puts n
+        dotFile.puts "\n\n  //  4 NODES --------------------------------------------------------------------"
+        @nodes.each do |n|
+          dotFile.puts n.dotLabel
         end
         dotFile.puts "\n\n  //  5--------------------------------------------------------------------"
-        emitTypes(      edges,   dotFile )
-        rankRootFields( dotFile, rootFields )
+        emitTypes(      dotFile )
         closeDot(       dotFile, twb )
-        # renderPng(twb.name,dotFileName)
-        # renderPdf(twb.name,dotFileName)
+        emit "Rendering DOT file - #{twb}"
         renderDot(twb,dotFileName,'pdf')
         renderDot(twb,dotFileName,'png')
         renderDot(twb,dotFileName,'svg')
-        emitEdges edges
+        # emitEdges
       end
 
+      def cypher twbName
+        cypher = Twb::Util::Cypher.new
+        cypher.fileName = "#{twbName}.calcFields"
+        cypher.nodes = @nodes
+        cypher.edges = @edges
+        cypher.render
+      end
 
-      def graphEdges twb, edges
-        graphFile = File.new(twb + '.cypher', 'w')
-        # graphFile.puts "OKEY DOKE, graphing away"
-        cypherCode = Set.new
-        edges.each do |edge|
-          cypherCode.add edge.from.cypherCreate
-          cypherCode.add edge.to.cypherCreate
-          cypherCode.add edge.cypherCreate
-        end
-        cypherCode.each do |cc|
-          graphFile.puts cc
-        end
-        graphFile.puts "\nreturn *"
-        graphFile.close unless graphFile.nil?
-        @imageFiles << File.basename(graphFile)
+      def cypherPy twbName
+        cypher = Twb::Util::CypherPython.new
+        cypher.fileName = "#{twbName}.calcFields"
+        cypher.nodes = @nodes
+        cypher.edges = @edges
+        cypher.render
       end
 
-      def emitEdges edges
+      # def graphEdges twb
+        #   # graphFile = File.new(twb + '.cypher', 'w')
+        #   # # graphFile.puts "OKEY DOKE, graphing away"
+        #   # cypherCode = Set.new
+        #   # @edges.each do |edge|
+        #   #   cypherCode.add edge.from.cypherCreate
+        #   #   cypherCode.add edge.to.cypherCreate
+        #   #   cypherCode.add edge.cypherCreate
+        #   # end
+        #   # cypherCode.each do |cc|
+        #   #   graphFile.puts cc
+        #   # end
+        #   # graphFile.puts "\nreturn *"
+        #   # graphFile.close unless graphFile.nil?
+        #   # @imageFiles << File.basename(graphFile)
+      # end
+
+      def emitEdges
         emit  "  %-15s    %s" % ['type', 'Edge']
         emit  "  %-15s    %s" % ['-'*15, '-'*35]
-        edges.each do |edge|
+        @edges.each do |edge|
           emit  "  %-15s    %s" % [edge.from.type, edge.from]
           emit  "  %-15s    %s" % [edge.to.type,   edge.to]
           emit  "\n "
         end
       end
 
-      def emitTypes edges, dotFile
+      def emitTypes dotFile
         typedNodes = {}
         dotFile.puts "\n\n  //  2--------------------------------------------------------------------"
-        edges.each do |edge|
+        @edges.each do |edge|
           emit  "   EDGE  :: #{edge}"
           loadNodeType typedNodes, edge.from
           loadNodeType typedNodes, edge.to
         end
         typedNodes.each do |type, nodes|
-          emit  "+++++++++ typedNodes of '#{type}''  "
-          nodes.each do |node|
-            emit  "           -n- #{node}"
-          end
-          rankSame(dotFile, type, nodes) unless type == :CalculatedField
+          # emit  "+++++++++ typedNodes of '#{type}''  "
+          # nodes.each do |node|
+          #   emit  "           -n- #{node}"
+          # end
+          rankSame(dotFile, type, nodes) unless type.eql? 'CalculatedField' # == :CalculatedField
         end
         # labelTypes dotFile, edges
       end
@@ -429,12 +427,17 @@ DOTHEADER
         set[type].add node
       end
 
+      @@unrankedTypes = ['CalculationField']
       def rankSame dotFile, type, nodes
+        return if @@unrankedTypes.include? type.to_s
+        @lines = SortedSet.new
+        nodes.each do |node|
+          @lines << node.id
+        end
         dotFile.puts "\n  // '#{type}' --------------------------------------------------------------------"
         dotFile.puts "\n  {rank=same "
-        # dotFile.puts "     \"#{type}\"  [shape=\"box3d\" style=\"filled\" ]" unless ''.eql? type # [shape=\"box3d\"  style=\"filled\" ]\"" unless label.equal? ''
-        nodes.each do |node|
-          dotFile.puts "     \"#{node.id}\""
+        @lines.each do |line|
+          dotFile.puts "     \"#{line}\""
         end
         dotFile.puts "  }"
       end
@@ -443,15 +446,16 @@ DOTHEADER
         dotFile.puts "\n  // Unreferenced (root) Calculated Fields -----------------------------------------"
         dotFile.puts "\n  {rank=same "
         dsRootFields.each do |rf|
+          emit "ROOT FIELD: #{rf.class}  :: #{rf}"
           dotFile.puts "     \"#{rf}\""
         end
         dotFile.puts "  }"
       end
 
 
-      def labelTypes dotFile, edges
+      def labelTypes dotFile
         fromTos = Set.new
-        edges.each do |edge|
+        @edges.each do |edge|
           # fromTos.add "\"Alien Data Source\"  -> \"Alien Data Source\""
           fromTos.add "\"#{edge.from.type}\""
           fromTos.add "\"#{edge.to.type}\""
@@ -467,26 +471,11 @@ DOTHEADER
         dotFile.puts '   }'
       end
 
-
-      def emit(local=@localEmit, stuff)
-          # puts "\nstuff.class #{stuff.class}  :: #{stuff}" if local
-          if stuff.is_a? String then
-            lines = stuff.split(/\n/)
-            lines.each do |line|
-              @logger.debug "#{@emitPrefix}#{line}"
-                       puts "#{@emitPrefix}#{line}" if local
-            end
-          else
-              @logger.debug "#{@emitPrefix}#{stuff}"
-                       puts "#{@emitPrefix}#{stuff}" if local
-          end
-      end
-
-
       def initDot twb
-        dotFile = File.open("#{twb}#{@@processName}.dot",'w')
+        dotFileName = docFile("#{twb}#{@@processName}.dot")
+        dotFile = File.open(dotFileName,'w')
         dotFile.puts @@dotHeader
-        return dotFile
+        return {:file => dotFile, :name => dotFileName}
       end
 
       def closeDot dotFile, twb
@@ -511,12 +500,12 @@ DOTHEADER
 
 
       def renderDot twb, dot, format
-        emit  "Rendering DOT file\n  - #{twb}\n  - #{dot}\n  - #{format}"
         imageType = '-T' + format
-        imageFile  = twb + @@processName + 'Graph.' + format
-        imageParam = '-o' + imageFile
-        emit  "system #{@@gvDotLocation} #{imageType} #{imageParam} #{dot}"
-               # system  @@gvDotLocation,   imageType,   imageParam,   dot
+        imageFile  = './ttdoc/' + twb + @@processName + 'Graph.' + format
+        imageParam = '-o"' + imageFile + '"'
+        emit  "system  #{@@gvDotLocation} #{imageType} #{imageParam} \"#{dot}\""
+               system "#{@@gvDotLocation} #{imageType} #{imageParam} \"#{dot}\""
+        emit  "           - #{imageFile}"
         @imageFiles << imageFile
         return imageFile
       end