datashift 0.9.0 → 0.10.0

data/lib/applications/apache_poi_extensions.rb

@@ -0,0 +1,62 @@
+# Copyright:: Autotelik Media Ltd
+# Author ::   Tom Statter
+# Date ::     July 2010
+# License::   
+#
+#
+if(DataShift::Guards::jruby?)
+  
+  require 'java'
+  require "poi-3.7-20101029.jar"
+      
+  # Extend the Poi classes with some syntactic sugar
+  
+  class Java::OrgApachePoiHssfUsermodel::HSSFSheet
+    def name() 
+      getSheetName
+    end
+    
+    def num_rows
+      getPhysicalNumberOfRows
+    end
+  
+  end
+  
+  class Java::OrgApachePoiHssfUsermodel::HSSFRow
+    
+    include RubyPoiTranslations
+    
+    include Enumerable
+        
+    def []( column)
+      cell_value( get_or_create_cell( column )  )
+    end
+    
+    def []=( column, value )
+      get_or_create_cell(column, value).setCellValue((value.to_s || ""))
+    end
+  
+    def get_or_create_cell( column, value = nil )
+      if(value)
+        java_send(:getCell, [Java::int], column) || createCell(column, poi_cell_type(value))
+      else
+        java_send(:getCell, [Java::int], column) || java_send(:createCell, [Java::int], column)
+      end
+    end
+    
+    def idx 
+      getRowNum() 
+    end
+    
+    # Iterate over each column in the row and yield on the cell
+    def each(&block) 
+      cellIterator.each {|c| yield cell_value(c) }        
+    end
+     
+    # TODO
+    # for min, max and sort from enumerable need <=>
+   # def <=> end
+     
+  end
+  
+end

data/lib/applications/excel.rb

@@ -0,0 +1,78 @@
+# Copyright:: Autotelik Media Ltd
+# Author ::   Tom Statter
+# Date ::     July 2010
+# License::   
+#
+# Details::   A wrapper around creating and directly manipulating Excel files. 
+#             Acts as proxy over main Ruby gem spreadsheet and our own JRuby only implementation using Apache POI
+#             Aim is to make it seamless to switch between any Excel implementation
+#             
+#             http://spreadsheet.rubyforge.org/GUIDE_txt.html
+#
+require 'guards'
+
+module ExcelProxy
+  # Returns the current proxy class
+  def self.proxy_class
+    if(DataShift::Guards.jruby?)
+      require 'jexcel_file'
+      JExcelFile
+    else
+      require 'spreadsheet'
+      require 'spreadsheet_extensions'
+      Spreadsheet
+    end
+  end
+  
+  def self.proxy_object
+    if(DataShift::Guards.jruby?)
+      ExcelProxy::proxy_class.new
+    else
+      ExcelProxy::proxy_class::Workbook.new
+    end
+  end
+end
+
+class Excel #< BasicObject
+
+  def initialize()
+    @excel_class = ExcelProxy::proxy_class
+    @excel = ExcelProxy::proxy_object
+  end
+   
+  # Forward all undefined methods to the wrapped Excel object.
+  def method_missing(method, *args, &block)
+    #puts @excel.class, method, args.inspect
+      
+    if(@excel.respond_to?(method))
+      @excel.send(method, *args, &block)
+    elsif(@excel.worksheets.last.respond_to?(method))  # active_worksheet doesn't work so use the latest
+      @excel.worksheets.last.send(method, *args, &block) 
+    elsif(@excel_class.respond_to?(method))
+      if(method == :open || method == 'open')
+        @excel = @excel_class.send(method, *args, &block)
+      else
+        @excel_class.send(method, *args, &block)
+      end
+    else            
+      super
+    end
+  end
+
+  def self.method_missing(method, *args, &block)           
+    @excel_class.send(method, *args, &block)
+  end
+      
+  # Returns +true+ if _obj_ responds to the given method. Private methods are included in the search
+  # only if the optional second parameter evaluates to +true+.
+  def respond_to?(method, include_private=false)
+    super || @excel.respond_to?(method, include_private)
+  end
+  
+  # without this can't get at any defined modules etc
+  # 
+  def self.const_missing(name)
+    ::Object.const_get(name)
+  end
+      
+end

data/lib/applications/excel_base.rb

@@ -0,0 +1,65 @@
+# To change this template, choose Tools | Templates
+# and open the template in the editor.
+
+module ExcelBase
+      
+  def sanitize_sheet_name( name )
+    name.gsub(/[\[\]:\*\/\\\?]/, '')
+  end
+  
+  # Helpers for dealing with Active Record models and collections
+  
+  def ar_to_headers( records )
+    return if( !records.first.is_a?(ActiveRecord::Base) || records.empty?)
+      
+    headers = records.first.class.columns.collect( &:name )    
+    set_headers( headers )
+  end
+      
+    
+  # Pass a set of AR records
+  def ar_to_xls(records, options = {})
+    return if( ! records.first.is_a?(ActiveRecord::Base) || records.empty?)
+      
+    row_index = 
+      if(options[:no_headers])
+      0
+    else
+      ar_to_headers( records )
+      1
+    end
+      
+    records.each do |record|
+      create_row(row_index)
+ 
+      ar_to_xls_row(0, record)
+        
+      row_index += 1
+    end
+  end
+   
+
+  # Save data from an AR record to the current row, based on the record's columns [c1,c2,c3]
+  # Returns the number of the final column written to  
+  def ar_to_xls_row(row, start_column, record)
+    return unless( record.is_a?(ActiveRecord::Base))
+        
+    column = start_column
+    record.class.columns.each do |connection_column|    
+      ar_to_xls_cell(row, column, record, connection_column)
+      column += 1
+    end
+    column
+  end
+  
+  def ar_to_xls_cell(row, column, record, connection_column)  
+    begin
+      datum = record.send(connection_column.name)
+
+      self[row, column] = datum
+    rescue => e
+      puts "Failed to export #{datum} from #{connection_column.inspect} to column #{column}"
+      puts e, e.backtrace
+    end
+  end
+end

data/lib/applications/jexcel_file.rb

@@ -0,0 +1,222 @@
+# Copyright:: Autotelik Media Ltd
+# Author ::   Tom Statter
+# Date ::     July 2010
+# License::   
+#
+# A wrapper around creating and directly manipulating Excel files. 
+#
+# i.e Create and populate XSL files
+# 
+# jar added to class path in manifest - 'poi-3.5-beta4-20081128.jar'
+#
+if(DataShift::Guards::jruby?)
+  
+  require 'java'
+  
+  require 'excel_base'
+  require 'ruby_poi_translations'
+  
+  class JExcelFile
+    
+    include ExcelBase
+    
+    include RubyPoiTranslations
+    extend RubyPoiTranslations
+        
+    include Enumerable
+    
+    include_class 'org.apache.poi.hssf.util.HSSFColor'
+    java_import 'org.apache.poi.poifs.filesystem.POIFSFileSystem'
+
+    java_import 'org.apache.poi.hssf.usermodel.HSSFCell'
+    java_import 'org.apache.poi.hssf.usermodel.HSSFWorkbook'
+    java_import 'org.apache.poi.hssf.usermodel.HSSFCellStyle'
+    java_import 'org.apache.poi.hssf.usermodel.HSSFDataFormat'
+    java_import 'org.apache.poi.hssf.usermodel.HSSFClientAnchor'
+    java_import 'org.apache.poi.hssf.usermodel.HSSFRichTextString'
+
+    attr_accessor :workbook, :row, :date_style
+    attr_reader   :sheet, :current_sheet_index
+
+    # NOTE: this is the POI 3.7 HSSF maximum rows
+    @@maxrows = 65535
+
+    def self.maxrows
+      return @@maxrows
+    end
+  
+    def self.date_format
+      HSSFDataFormat.getBuiltinFormat("m/d/yy h:mm")
+    end
+    
+    def self.open(filename)
+      HSSFWorkbook.new(FileInputStream.new(filename))
+    end
+    
+    # NOTES :
+    #   The HSSFWorkbook uses 0 based indexes
+  
+    def initialize()
+      @workbook = HSSFWorkbook.new
+      
+      @sheet = nil
+      @current_sheet_index = 0
+      
+      # The @patriarchs hash is a workaround because HSSFSheet.getDrawingPatriarch()
+      # causes a lot of issues (if it doesn't throw an exception!)
+      @patriarchs = Hash.new
+      
+      @date_style = nil
+    end
+   
+    def open(filename)
+      @workbook = JExcelFile.open(filename)
+      
+      @date_style = @workbook.createCellStyle
+      @date_style.setDataFormat( JExcelFile::date_format )
+      
+      activate_sheet(0)
+      @workbook
+    end
+    
+    # Create and return a new worksheet. 
+    # Not set to the active worksheet
+    
+    def create_worksheet( options = {} )
+      sheet_name = options[:name]
+      
+      @workbook = HSSFWorkbook.new() if @workbook.nil?
+            
+      unless(sheet_name)
+        i = 0
+        begin
+          sheet_name = "Worksheet#{i += 1}"
+        end while(@workbook.getSheetIndex(sheet_name) >= 0) # there is no hard limit to no of sheets in Excel but at some point you will run out of memory!
+        
+        return create_sheet_and_set_styles( sheet_name )
+      else 
+        if (@workbook.getSheetIndex(sheet_name) < 0)  #Check sheet doesn't already exist
+          return create_sheet_and_set_styles( sheet_name )
+        else
+          activate_sheet(sheet_name)
+        end
+      end
+    end
+
+    # Set the supplied sheet index or name, as the active sheet and return it.
+    # If no such sheet return current sheet
+    def activate_sheet(term)
+
+      if(@workbook)
+        x = term.is_a?(String) ? @workbook.getSheetIndex(term.to_java(java.lang.String)) : term
+        @sheet = worksheet(x)
+        
+        if( @sheet )
+          @current_sheet_index = x
+          @workbook.setActiveSheet(@current_sheet_index)
+          @sheet = @workbook.getSheetAt(@current_sheet_index)
+          @sheet.setActive(true)
+        end
+      end
+      @sheet
+    end
+
+    # Return a sheet by index
+    def worksheet( index )  
+      if(@workbook)
+        x = index.is_a?(String) ? @workbook.getSheetIndex(index.to_java(java.lang.String)) : index
+        return @workbook.getSheetAt(x) 
+      end
+      nil
+    end
+    
+    def worksheets
+      (0...@workbook.getNumberOfSheets).collect { |i| @workbook.getSheetAt(i) }
+    end
+
+    # Create new row (indexing in line with POI usage, start 0)
+    def create_row(index)
+      return nil if @sheet.nil?
+      raise "BAD INDEX: Row indexing starts at 0" if(index < 0)
+      @row = @sheet.createRow(index)
+      @row
+    end
+    
+    def num_rows
+      @sheet.getPhysicalNumberOfRows
+    end
+
+    # Process each row. Row type is org.apache.poi.hssf.usermodel.HSSFRow
+        
+    # Currently ignores skip argument - TODO - this is how spreadsheet gem works
+    # #each iterates over all used Rows (from the first used Row until but omitting the first unused Row, see also #dimensions)
+    # If the argument skip is given, 
+    # #each iterates from that row until but omitting the first unused Row, effectively skipping the first skip Rows from the top of the Worksheet. 
+
+    def each(skip = nil, &block) 
+      @sheet.rowIterator.each(&block)
+    end
+
+    def row( index )
+      @sheet.getRow(index) || create_row(index)
+    end
+         
+    # Get the enriched value of the Cell at row, column.
+    def cell(row, column) 
+      row = row(row)
+      cell_value( row.get_or_create_cell( column )  )
+    end
+
+    # Get the enriched value of the Cell at row, column. 
+    def [](row, column) 
+      cell(row, column) 
+    end
+
+    def []=(row, column, value)
+      set_cell(row, column, value)
+    end
+    
+    def set_cell(row, column, value)
+      @row = row(row)
+      
+      @row.get_or_create_cell(column, value).setCellValue((value || ""))
+    end
+    
+    def write( filename = nil )
+      filename.nil? ? file = @filepath : file = filename
+      out = FileOutputStream.new(file)
+      @workbook.write(out) unless @workbook.nil?
+      out.close
+    end
+
+    alias_method :save, :write
+    
+    def save_to_text( filename )
+      File.open( filename, 'w') {|f| f.write(to_s) }
+    end
+
+    def to_s
+      outs = ByteArrayOutputStream.new
+      @workbook.write(outs);
+      outs.close();
+      String.from_java_bytes(outs.toByteArray)
+    end
+    
+    private
+       
+    def create_sheet_and_set_styles( sheet_name )
+      @sheet = @workbook.createSheet( sanitize_sheet_name(sheet_name) )
+
+      @patriarchs.store(sheet_name, @sheet.createDrawingPatriarch())
+     
+      @date_style = @workbook.createCellStyle
+      @date_style.setDataFormat( JExcelFile::date_format )
+      @sheet
+    end
+    
+  end
+  
+  require 'jexcel_file_extensions'
+  require 'apache_poi_extensions'
+  
+end

data/lib/applications/jexcel_file_extensions.rb

@@ -0,0 +1,244 @@
+# Copyright:: Autotelik Media Ltd
+# Author ::   Tom Statter
+# Date ::     July 2010
+# License::   
+#
+# Details::   A helper module providing shortcuts for manipulating Excel files. 
+# 
+
+if(DataShift::Guards::jruby?)
+    
+  JExcelFile.class_eval do
+    
+    include_class 'org.apache.poi.hssf.util.HSSFColor'
+    
+    # Return the current active sheet
+    # If term supplied find sheet and set active
+    # 
+    def active_worksheet(term = nil)
+      if( term.nil? )
+        @sheet ||= @workbook.getSheetAt(@current_sheet_index)
+      else 
+        activate_sheet(term)
+      end if(@workbook)
+      @sheet
+    end
+    
+    #  Populate a row  of cells with data in an array 
+    #  where the co-ordinates relate to row/column start position
+    #    
+    def set_row( row, col, data, sheet_num = nil)
+    
+      sheet(sheet_num)
+      
+      create_row(row)
+      
+      column = col
+      data.each do |datum|
+        set_cell(row, column, datum)
+        column += 1
+      end
+    end
+    
+    def headers=( headers )  
+      headers.each_with_index do |header, column|
+        self[0, column] = (header || "")
+      end
+    end
+    
+    # Convert array into a header row
+    def set_headers(headers, apply_style = nil)
+      create_row(0)
+      return if headers.empty?
+    
+      style = apply_style || header_style()
+
+      headers.each_with_index do |datum, i|
+        c = @row.createCell(i, poi_cell_type(datum))
+        c.setCellValue(datum)
+        c.setCellStyle(style)
+      end
+    end
+    
+    attr_writer :header_background_colour
+    
+    def header_background_colour
+      @header_background_colour ||= org.apache.poi.hssf.util.HSSFColor::LIGHT_CORNFLOWER_BLUE.index        
+    end
+    
+    def header_style
+      return @header_style if @header_style
+      @header_style = @workbook.createCellStyle();
+      @header_style.setBorderTop(6) # double lines border
+      @header_style.setBorderBottom(1) # single line border
+      @header_style.setFillBackgroundColor(header_background_colour)
+      
+      @header_style
+    end
+    
+    def add_comment( cell, text )
+      raise "Please supply valid HSSFCell" unless cell.respond_to?('setCellComment')
+      return if @sheet.nil?
+
+      patriarch = @patriarchs[@sheet.getSheetName()]
+
+      anchor = HSSFClientAnchor.new(100, 50, 100, 50, cell.getColumnIndex(), cell.getRowIndex(), cell.getColumnIndex()+3, cell.getRowIndex()+4)
+      comment = patriarch.createCellComment(anchor)
+
+      comment_text = HSSFRichTextString.new(text)
+      comment.setString(comment_text)
+      comment.setAuthor("Mapping")
+
+      cell.setCellComment(comment)
+    end
+    
+    # Get a percentage style
+    def getPercentStyle()
+      if (@percentCellStyle.nil? && @workbook)
+        @percentCellStyle = @workbook.createCellStyle();
+        @percentCellStyle.setDataFormat(HSSFDataFormat.getBuiltinFormat("0.00%"));
+      end
+      return @percentCellStyle
+    end
+
+    # Auto size either the given column index or all columns
+    def autosize(column = nil)
+      return if @sheet.nil?
+      if (column.kind_of? Integer)
+        @sheet.autoSizeColumn(column)
+      else
+        @sheet.getRow(0).cellIterator.each{|c| @sheet.autoSizeColumn(c.getColumnIndex)}
+      end
+    end
+    
+    def create_freeze_pane(row=1, column=0)
+      return if @sheet.nil?
+      @sheet.createFreezePane(row, column)
+    end
+
+    
+    # Use execute to run sql query provided
+    # and write to a csv file (path required)
+    # header row is optional but default is on
+    # Auto mapping of specified columns is optional
+    # @mappings is a hash{column => map} of columns to a map{old_value => new_value}
+    def results_to_sheet( results, sheet, mappings=nil, header=true)
+      numrows = results.length
+      sheet_name = sheet
+
+      if (numrows == 0)
+        log :info, "WARNING - results are empty nothing written to sheet: #{sheet}"
+        return
+      end
+
+      #Check if we need to split the results into seperate sheets
+      if (numrows > @@maxrows )
+        startrow = 0
+        while (numrows > 0)
+          # Split the results and write to a new sheet
+          next_results = results.slice(startrow, @@maxrows > numrows ? numrows : @@maxrows)
+          self.results_to_sheet(next_results, "#{sheet_name}", mappings, header) if next_results
+
+          # Increase counters
+          numrows -= next_results.length
+          startrow += next_results.length
+          sheet_name += 'I'
+        end
+      else
+        log :info, "Writting #{numrows} rows to : #{sheet_name}"
+        # Create required sheet
+        self.create(sheet)
+
+        row_index = self.num_rows
+        # write header line
+        if (header && row_index==0 )
+          header_row = @sheet.createRow(row_index)
+          cell_index = 0
+          results[0].keys.each{ |h|
+            header_row.createCell(cell_index).setCellValue("#{h}")
+            @sheet.setDefaultColumnStyle(cell_index, self.getPercentStyle) if "#{h}".include? '%'
+            cell_index += 1
+          }
+          # Freeze the header row
+          @sheet.createFreezePane( 0, 1, 0, 1 )
+          row_index += 1
+        end
+
+        # write_results
+        results.each{ |row|
+          sheet_row = @sheet.createRow(row_index)
+          cell_index = 0
+          row.each{|k,v|
+            celltype = v.kind_of?(Numeric) ? HSSFCell::CELL_TYPE_NUMERIC : HSSFCell::CELL_TYPE_STRING
+            cell = sheet_row.createCell(cell_index, celltype)
+
+            v.nil? ? value = "<NIL>" : value = v
+
+            cell.setCellValue(value)
+
+            cell_index +=1
+          }
+          #puts "#{sheet}: written row #{row_index}"
+          row_index +=1
+        }
+      end
+    end
+    
+    module ExcelHelper
+      require 'java'
+
+      java_import 'org.apache.poi.poifs.filesystem.POIFSFileSystem'
+      java_import 'org.apache.poi.hssf.usermodel.HSSFCell'
+      java_import 'org.apache.poi.hssf.usermodel.HSSFWorkbook'
+      java_import 'org.apache.poi.hssf.usermodel.HSSFCellStyle'
+      java_import 'org.apache.poi.hssf.usermodel.HSSFDataFormat'
+      java_import 'java.io.ByteArrayOutputStream'
+      java_import 'java.util.Date'
+
+      # ActiveRecord Helper - Export model data to XLS file format
+      #
+      def to_xls(items=[])
+
+        @excel = ExcelFile.new(items[0].class.name)
+    
+        @excel.create_row(0)
+  
+        sheet = @excel.sheet
+
+        # header row
+        if !items.empty?
+          row = sheet.createRow(0)
+          cell_index = 0
+          items[0].class.columns.each do |column|
+            row.createCell(cell_index).setCellValue(column.name)
+            cell_index += 1
+          end
+        end
+
+        # value rows
+        row_index = 1
+        items.each do |item|
+          row = sheet.createRow(row_index);
+
+          cell_index = 0
+          item.class.columns.each do |column|
+            cell = row.createCell(cell_index)
+            if column.sql_type =~ /date/ then
+              millis = item.send(column.name).to_f * 1000
+              cell.setCellValue(Date.new(millis))
+              cell.setCellStyle(dateStyle);
+            elsif column.sql_type =~ /int/ then
+              cell.setCellValue(item.send(column.name).to_i)
+            else
+              value = item.send(column.name)
+              cell.setCellValue(item.send(column.name)) unless value.nil?
+            end
+            cell_index += 1
+          end
+          row_index += 1
+        end
+        @excel.to_s
+      end
+    end
+  end
+end