twb 1.0.5 → 1.9.1

data/lib/twb/analysis/CalculatedFields/MarkdownEmitter.rb

@@ -0,0 +1,71 @@
+#  calculatedfieldsanalyzer.rb - this Ruby script Copyright 2017 Christopher Gerrard
+#
+#  This program is free software: you can redistribute it and/or modify
+#  it under the terms of the GNU General Public License as published by
+#  the Free Software Foundation, either version 3 of the License, or
+#  (at your option) any later version.
+#
+#  This program is distributed in the hope that it will be useful,
+#  but WITHOUT ANY WARRANTY; without even the implied warranty of
+#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#  GNU General Public License for more details.
+#
+#  You should have received a copy of the GNU General Public License
+#  along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+require 'twb'
+
+module Twb
+module Analysis
+module CalculatedFields
+
+  class MarkdownEmitter
+
+    attr_reader :docFileName
+
+    def initialize
+      # puts ""
+    end
+
+    def processTwb twb
+       twb = File.basename(twb)
+      @twb = Twb::Workbook.new twb
+      @docFileName = twb + '.CalculatedFields.md'
+      @docFile     = File.open(@docFileName,'w')
+      @docFile.puts "## #{twb}"
+      dsNames = @twb.datasourceUINames
+      @docFile.puts "#{dsNames.length} Data Sources"
+      dsNames.each do |dsn|
+        ds = @twb.datasource dsn
+      #   puts "  => #{ds.uiname} "
+      # end
+      # dss = @twb.datasources
+      # dss.each do |ds|
+        @docFile.puts "### #{ds.uiname}"
+        calcFields = ds.calculatedFieldsMap.sort_by { |fldName,calc| fldName }
+        calcFields.each do |fldname, field|
+          calculation = field.calculation
+          @docFile.puts "\n##### #{fldname} "
+          @docFile.puts "```"
+          if calculation.has_formula 
+            @docFile.puts calculation.formulaResolved
+            @docFile.puts "\n -- Fields  --" unless calculation.calcFields.empty?
+            refFields = SortedSet.new
+            calculation.calcFields.each do |cf|
+              fds =  if cf.dataSourceRef == :remote then "<<#{cf.dataSource}>>" else '' end
+              refFields.add "#{cf.uiName} \t #{fds}"
+            end
+            refFields.each do |rf|
+              @docFile.puts "    #{rf}"
+            end
+            @docFile.puts "```"
+          end
+        end
+      end
+    end
+
+  end # class MarkdownEmitter
+
+end   # nodule CalculatedFields
+end   # module Analysis
+end   # module Twb

data/lib/twb/analysis/DataSources/DataSourceTableFieldsCSVEmitter.rb

@@ -0,0 +1,117 @@
+#  calculatedfieldsanalyzer.rb - this Ruby script Copyright 2017 Christopher Gerrard
+#
+#  This program is free software: you can redistribute it and/or modify
+#  it under the terms of the GNU General Public License as published by
+#  the Free Software Foundation, either version 3 of the License, or
+#  (at your option) any later version.
+#
+#  This program is distributed in the hope that it will be useful,
+#  but WITHOUT ANY WARRANTY; without even the implied warranty of
+#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#  GNU General Public License for more details.
+#
+#  You should have received a copy of the GNU General Public License
+#  along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+require 'twb'
+
+module Twb
+module Analysis
+module DataSources
+
+  class DataSourceTableFieldsCSVEmitter
+
+    attr_reader :csvFileName, :csvRecords
+    attr_reader :dsCount,     :tablesCount, :fieldsCount  
+
+    @@csvFileName        = 'DataSourceTableFields.csv'
+    @csvFileDescription = 'Contains CSV records, each containing the details for an individual Field contained in a table in the Data Source. Data Sources are identified within their host Workbook. '
+
+    @@csvHeader = ['Record #',
+                   'Workbook',           
+                   'Workbook Dir',
+                   'Data Source',
+                   'Data Source (tech)',
+                   'Table',
+                   'Field',
+                   'Field (code)',
+                   'Field (table)',
+                  ]
+
+
+    def initialize
+      @csvFile = CSV.open(@@csvFileName, 'w')
+      @csvFile << @@csvHeader
+      @outputs = Set.new
+      @outputs << @@csvFileName
+      @dsCount      = 0
+      @tablesCount  = 0
+      @fieldsCount  = 0
+      @csvRecords   = Set.new
+    end
+
+    def doc
+      {
+        :filedoc => [
+                     { :file        => @@csvFileName,
+                       :description => @csvFileDescription,
+                       :header      => @@csvHeader 
+                     }
+                    ],
+        :outputs => @outputs
+      }
+    end
+
+    def self.csvHeader
+      @@csvHeader
+    end
+
+    def self.csvFileType
+      @@csvFileName
+    end
+
+    def outputs
+      @outputs
+    end
+
+    def processTwb twb
+      @recNum = 0
+      @twb    = twb if twb.instance_of? Twb::Workbook
+      @twb    = Twb::Workbook.new(twb) if twb.instance_of? String
+      # --
+      dss = @twb.datasources
+      dss.each do |ds|
+        tables = Set.new
+        @dsCount += 1
+        ds.node.xpath('./connection/cols/map').each do |cnode|
+          # puts cnode
+          key         = cnode.attribute('key').text
+          codename    = key.gsub(/^\[|\]$/,'')
+          fielduiname = ds.fieldUIName(codename)
+          value       = cnode.attribute('value').text.gsub(/^\[|\]$/,'')
+          parts       = value.split('].[')
+          # puts "%-30s  %-45s  %s \n " % [fielduiname, value, parts]
+          csvRecord = [ @fieldsCount += 1,
+                        @twb.name,
+                        @twb.dir,
+                        ds.uiname,
+                        ds.name,
+                        parts[0],
+                        fielduiname,
+                        codename,
+                        parts[1] 
+                      ]
+          tables.add parts[0]
+          @csvRecords.add csvRecord
+          @csvFile << csvRecord
+        end
+        @tablesCount += tables.length
+      end
+    end
+
+  end # class CSVEmitter
+
+
+end   # module DataSources
+end   # module Analysis
+end   # module Twb

data/lib/twb/analysis/Sheets/WorksheetDataStructureCSVEmitter.rb

@@ -0,0 +1,192 @@
+#  calculatedfieldsanalyzer.rb - this Ruby script Copyright 2017 Christopher Gerrard
+#
+#  This program is free software: you can redistribute it and/or modify
+#  it under the terms of the GNU General Public License as published by
+#  the Free Software Foundation, either version 3 of the License, or
+#  (at your option) any later version.
+#
+#  This program is distributed in the hope that it will be useful,
+#  but WITHOUT ANY WARRANTY; without even the implied warranty of
+#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#  GNU General Public License for more details.
+#
+#  You should have received a copy of the GNU General Public License
+#  along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+require 'twb'
+require 'csv'
+
+module Twb
+module Analysis
+module Sheets
+
+  class  WorksheetDataStructureCSVEmitter
+
+    # attr_reader :csvFieldsFileName, :csvRecords
+    attr_reader :sheetNames,  :sheetCount 
+
+    @@csvSheetPanesFileType = 'WorksheetPanes'
+    @@csvSheetPanesHeader   = ['Record #',
+                               'Workbook',           
+                               'Workbook Dir',
+                               'Worksheet',
+                               'Pane #',
+                               'Pane #of',
+                               'Axis Type',
+                               'Axis Field Code',
+                               'Axis Field Data Source',
+                               'Axis Field Data Source (tech)',
+                               'Axis Field Prefix',
+                               'Axis Field Name',
+                               'Axis Field Suffix'
+                              ]
+
+    @@csvSheetFieldsFileType = 'WorksheetFieldsStructure'
+    @@csvSheetFieldsHeader   = ['Record #',
+                                'Workbook',           
+                                'Workbook Dir',
+                                'Worksheet',
+                                'Pane #',
+                                'Data Source',
+                                'Data Source (tech)',
+                                'Function',
+                                'Field Code',
+                                'Field Name (tech)',
+                                'Field Prefix',
+                                'Field Suffix',
+                                'Position'
+                               ]
+
+
+    def initialize
+      @csvPanesFileName = "Twb#{@@csvSheetPanesFileType}.csv"
+      @csvPanesFile     = CSV.open(@csvPanesFileName,'w')
+      @csvPanesFile << @@csvSheetPanesHeader
+      # --
+      @csvFieldsFileName = "Twb#{@@csvSheetFieldsFileType}.csv"
+      @csvFieldsFile     = CSV.open(@csvFieldsFileName,'w')
+      @csvFieldsFile << @@csvSheetFieldsHeader
+      # --
+      @twbsCnt   = 0
+      @sheetsCnt = 0
+      @panesCnt  = 0
+      @fieldsCnt = 0
+    end
+
+    # def self.csvHeader
+    #   @@csvHeader
+    # end
+
+    # def self.csvFileType
+    #   @@csvFieldsFileType
+    # end
+
+    def processTwb twb
+      @twb    = case twb
+                when String        then Twb::Workbook.new(twb)
+                when Twb::Workbook then twb
+                else raise ArgumentError.new("ERROR: #{twb} must be a String or Workbook, is a #{twb.class}")
+                end
+      # --
+      @sheetNames = SortedSet.new
+      # --
+      sheets = @twb.worksheets
+      sheets.each do |sheet|
+        @sheetNames << sheet.name
+        processPanes    sheet
+        processRowsCols sheet
+      end
+    end
+
+    def processPanes sheet
+      pnodes   = sheet.node.xpath('.//panes/pane')
+      panesCnt = pnodes.length
+      pnodes.each do |node|
+        id = node.has_attribute?('id') ? node.attribute('id').text : '0'
+        #--
+        if    node.has_attribute?('x-axis-name')
+          axistype  = 'x'
+          field     = Twb::CodedField.new node.attribute('x-axis-name').text
+        elsif node.has_attribute?('y-axis-name')
+          axistype = 'y'
+          field     = Twb::CodedField.new node.attribute('y-axis-name').text
+        else 
+          axistype  = nil
+          field     = nil
+        end
+        #--
+        @csvPanesFile << [ @panesCnt+=1,
+                           @twb.name,                           # 'Workbook',           
+                           @twb.dir,                            # 'Workbook Dir',
+                           sheet.name,                          # 'Worksheet',
+                           id,                                  # 'Pane #',
+                           panesCnt,                            # 'Pane #of',
+                           axistype,                            # 'Axis Type',
+                           field.nil? ? ' ' : field.code,        # 'Axis Field Code',
+                           field.nil? ? ' ' : field.dataSource,  # 'Axis Field Code',
+                           field.nil? ? ' ' : field.prefix,      # 'Axis Field Prefix',
+                           field.nil? ? ' ' : field.name,        # 'Axis Field Name',
+                           field.nil? ? ' ' : field.suffix,      # 'Axis Field Suffix'
+                         ]
+        processPaneFields sheet.name, node, id
+      end
+    end
+
+      def processPaneFields sheet, node, paneID
+        encodedFields = node.xpath('.//encodings/*')
+        # puts "    flds : #{encodedFields.length} "
+        fieldPos = 0
+        encodedFields.each do |ef|
+          function = ef.name
+          field    = Twb::CodedField.new(ef.attribute('column').text)
+          dsuiname = @twb.datasource(field.dataSource).uiname
+          # puts "      ef : <<#{function}>>  #{ef}  -> #{field} "
+          @csvFieldsFile << [  @fieldsCnt+=1,     # 'Record #',
+                               @twb.name,         # 'Workbook',           
+                               @twb.dir,          # 'Workbook Dir',
+                               sheet,             # 'Worksheet',
+                               paneID,            # 'Pane #',
+                               dsuiname,          # 'Data Source',
+                               field.dataSource,  # 'Data Source (tech)',
+                               function,          # 'Function',
+                               field.code,        # 'Field Code',
+                               field.name,        # 'Field Name (tech)',
+                               field.prefix,      # 'Field Prefix',
+                               field.suffix,      # 'Field Suffix',
+                               fieldPos+=1
+                            ]
+        end
+
+      end
+
+      def processRowsCols sheet
+        emitRowColFields sheet, 'row', sheet.rowFields
+        emitRowColFields sheet, 'col', sheet.colFields
+      end
+
+      def emitRowColFields sheet, type, fields
+        fieldPos = 0
+        fields.each do |field|
+          dsuiname = @twb.datasource(field.dataSource).uiname
+          @csvFieldsFile << [  @fieldsCnt+=1,     # 'Record #',
+                               @twb.name,         # 'Workbook',           
+                               @twb.dir,          # 'Workbook Dir',
+                               sheet.name,        # 'Worksheet',
+                               nil,               # 'Pane #',
+                               dsuiname,          # 'Data Source',
+                               field.dataSource,  # 'Data Source (tech)',
+                               type,              # 'Function',
+                               field.code,        # 'Field Code',
+                               field.name,        # 'Field Name (tech)',
+                               field.prefix,      # 'Field Prefix',
+                               field.suffix,      # 'Field Suffix',
+                               fieldPos+=1
+                            ]
+        end
+      end
+
+  end # class WorksheetDataStructureCSVEmitter
+
+end   # module Sheets
+end   # module Analysis
+end   # module Twb

data/lib/twb/calculatedfield.rb

@@ -0,0 +1,47 @@
+#  Copyright (C) 2014, 2015, 2017  Chris Gerrard
+#
+#  This program is free software: you can redistribute it and/or modify
+#  it under the terms of the GNU General Public License as published by
+#  the Free Software Foundation, either version 3 of the License, or
+#  (at your option) any later version.
+#
+#  This program is distributed in the hope that it will be useful,
+#  but WITHOUT ANY WARRANTY; without even the implied warranty of
+#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#  GNU General Public License for more details.
+#
+#  You should have received a copy of the GNU General Public License
+#  along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+require 'nokogiri'
+
+module Twb
+
+  class CalculatedField
+
+    attr_reader :node,     :dataSource
+    attr_reader :caption,  :name, :uiname  
+    attr_reader :datatype, :role, :type  
+    attr_reader :calculation
+
+    def initialize(calcNode, datasource=nil)
+        @dataSource = datasource
+        @node       = calcNode
+        # -- 
+        @caption    = calcNode.attribute('caption').text if calcNode.has_attribute?('caption')
+        @name       = calcNode.attribute('name').text.gsub(/^\[/,'').gsub(/\]$/,'')
+        @uiname     = caption.nil? ? @name : @caption
+        # -- 
+        @datatype    = @node.attribute('datatype').text
+        @role        = @node.attribute('role').text
+        @type        = @node.attribute('type').text
+        # -- 
+        @calculation = Twb::FieldCalculation.new(self, datasource)
+    end
+
+    def to_s
+        "%s(%s) => %s" % [uiname, name, @calculation.formulaFlat]
+    end
+
+  end # class
+end # module

data/lib/twb/columnfield.rb

@@ -0,0 +1,94 @@
+#  Copyright (C) 2014, 2015  Chris Gerrard
+#
+#  This program is free software: you can redistribute it and/or modify
+#  it under the terms of the GNU General Public License as published by
+#  the Free Software Foundation, either version 3 of the License, or
+#  (at your option) any later version.
+#
+#  This program is distributed in the hope that it will be useful,
+#  but WITHOUT ANY WARRANTY; without even the implied warranty of
+#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#  GNU General Public License for more details.
+#
+#  You should have received a copy of the GNU General Public License
+#  along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+require 'nokogiri'
+
+module Twb
+
+  class ColumnField
+
+    include Comparable
+
+    # XML Attributes known to exist in TWB xml
+    # ---------------------------------------- 
+    # aggregation
+    # aggregation-param
+    # alias
+    # auto-column
+    # caption
+    # datatype
+    # datatype-customized
+    # default-format
+    # default-role
+    # default-type
+    # hidden
+    # layered
+    # name
+    # param-domain-type
+    # pivot
+    # role
+    # semantic-role
+    # SplitFieldIndex
+    # SplitFieldOrigin
+    # type
+    # user-datatype
+    # value
+    # visual-totals
+
+    attr_reader :node
+    attr_reader :name,       :caption,       :uiname
+    attr_reader :dataType,   :defaultFormat, :paramDomainType
+    attr_reader :role,       :type,          :value
+    attr_reader :alias,      :semanticRole,  :aggregation
+    attr_reader :autoColumn, :hidden,        :datatypeCustomized
+
+
+    def initialize fieldNode
+      @node               = fieldNode
+      @name               = load 'name'
+      @caption            = load 'caption'
+      @uiname             = @caption.nil? ? @name : @caption
+      @dataType           = load 'datatype'
+      @userDataType       = load 'user-datatype'
+      @defaultFormat      = load 'default-format'
+      @paramDomainType    = load 'param-domain-type'
+      @role               = load 'role'
+      @type               = load 'type'
+      @value              = load 'value'
+      @alias              = load 'alias'
+      @semanticRole       = load 'semantic-role'
+      @aggregation        = load 'aggregation'
+      @autoColumn         = load 'auto-column'
+      @hidden             = load 'hidden'
+      @datatypeCustomized = load 'datatype-customized'
+    end
+
+    def load nodeName
+      attr = @node.attribute(nodeName)
+      val  = attr.nil? ? nil : attr.text.strip.gsub(/^\[|\]$/,'')
+      return val
+    end
+
+    def to_s
+      @name
+    end
+
+    def <=>(other)
+      @name <=> other.name
+    end
+
+  end  # class ColumnField
+
+end