datashift 0.16.0 → 0.40.0

data/lib/exporters/exporter_base.rb

@@ -1,6 +1,5 @@
-# Copyright:: (c) Autotelik Media Ltd 2011
+# Copyright:: (c) Autotelik Media Ltd 2015
 # Author ::   Tom Statter
-# Date ::     Aug 2010
 # License::   MIT
 #
 #  Details::  Base class for Exporters, which provide services to export a Model
@@ -10,12 +9,13 @@ module DataShift
 
   class ExporterBase
 
-    attr_accessor :filename
-  
-    def initialize(filename)
-      @filename = filename
+    attr_accessor :configuration
+    attr_accessor :file_name
+
+    def initialize
+      @configuration = DataShift::Exporters::Configuration.call
     end
-    
+
   end
 
-end
+end

data/lib/generators/config_generator.rb

@@ -0,0 +1,218 @@
+# Copyright:: (c) Autotelik Media Ltd 2015
+# Author ::   Tom Statter
+# Date ::     Mar 2015
+# License::   MIT
+#
+# Details::   Generate an empty Configuration file, users can populate
+#             to configure their specific import and export
+#
+require 'generator_base'
+
+module DataShift
+
+  class ConfigGenerator < GeneratorBase
+
+    def self.title
+      @mappings_title ||= "datashift_mappings:\n"
+    end
+
+    attr_accessor :output_filename, :headers
+
+    attr_writer :import_template, :export_template
+
+    def initialize
+      super
+    end
+
+    def import_template
+      @import_template ||= File.join(DataShift.library_path, 'datashift/templates/import_export_config.erb')
+    end
+
+    def export_template
+      @export_template ||= File.join(DataShift.library_path, 'datashift/templates/import_export_config.erb')
+    end
+
+    # You can pass Transformations into the options
+    #
+    #     options[:defaults]
+    #     options[:overrides]
+    #     options[:substitutions]
+    #     options[:prefixes]
+    #     options[:postfixes]
+    #
+    # For example :
+    #
+    #   options = {
+    #     defaults: {'value_as_string': 'some default text', 'value_as_double': 45.467 }
+    #   }
+    #
+    def write_import(file_name, klass_or_name, options = {})
+      result = create_import_config(klass_or_name, options)
+
+      logger.info("Writing Import Config File [#{file_name}]")
+
+      File.open(file_name, 'w') { |f| f << result }
+
+      result
+    end
+
+    # N.B Gettign the YAML formatted correctly was nigh on impossible  in ERB,
+    # be careful of the spacing in the string sections here e.g dont use IDE auto spacing tools
+
+    # TODO: - How to better isolate these string template snippets
+
+    # Create an YAML ERB Configuration template for Importing.
+    # Includes available transformations and column mapping
+    #
+    # For other options See DataShift::Loaders::Configuration
+    #
+    def create_section(hash)
+      section = ''
+      (hash || { "#name": 'value' } ).each { |n, v| section += "        #{n}: #{v}\n" }
+      section
+    end
+
+    def create_import_config(klass_or_name, options = {})
+
+      klass = MapperUtils.ensure_class(klass_or_name)
+
+      @key = 'data_flow_schema'
+      @klass = klass.name.to_s
+
+      defaults_section = create_section(options[:defaults] )
+      overrides_section = create_section(options[:overrides] )
+      prefixes_section = create_section(options[:prefixes] )
+      postfixes_section = create_section(options[:postfixes] )
+
+      @substitutions = options[:substitutions] || {}
+
+      # operator => [rule , replacement]
+      substitutions_section = ''
+
+      @substitutions.each do |o, v|
+        raise BadConfig, 'Substitutions need be in format {operator: [rule, replacement]}' unless(v.is_a? Array)
+        substitutions_section += <<-EOS
+        #{o}:
+          - #{v.first}
+          - #{v.last}
+        EOS
+      end
+
+      @headers = Headers.klass_to_headers(klass)
+
+      nodes_section = <<-EOS
+    # Mappings between inbound column names and internal names
+    # are only required when datashift cannot guess the mapping itself
+    # It will automatically map headings like :
+    #  'Product properties' or 'Product_Properties', 'product Properties' etc to product_properties
+    nodes:
+EOS
+
+      @headers.each_with_index do |s, _i|
+        nodes_section += <<-EOS
+        - #{s}:
+            heading:
+               source: #{s}
+               presentation: #{s}
+        EOS
+      end
+
+      x = <<-EOS
+# YAML Configuration file for Datashift Import/Export
+#
+#{@key}:
+  #{@klass}:
+    defaults:
+#{defaults_section}
+    overrides:
+ #{overrides_section}
+    # Expects a tuple (list with 2 entries), the rule and the replacement
+    substitutions:
+#{substitutions_section}
+    prefixes:
+#{prefixes_section}
+    postfixes:
+#{postfixes_section}
+
+#{nodes_section}
+
+      EOS
+
+      # This was a nightmare to get proeprly formatted YAML
+      # Erubis::Eruby.new( File.read(import_template)).result(binding)
+
+      x
+    end
+
+    # FOR EXPORTERS
+
+    def write_export(file_name, klass_or_name, options = {})
+      result = export(klass_or_name, options)
+
+      logger.info("Writing Export Config File [#{config[:file]}]")
+
+      File.open(file_name, 'w') { |f| f << result }
+    end
+
+    # Create an YAML Configuration template for Exporting
+    # includes available transformations and column mapping
+    #
+    # For other options See DataShift::Exporters::Configuration
+    #
+    def export(klass_or_name, options = {})
+
+      @klass = MapperUtils.ensure_class(klass_or_name)
+
+      @title = @klass.name.to_s
+
+      @defaults = options[:defaults] || []
+
+      @overrides = options[:overrides] || []
+      @substitutions = options[:substitutions] || []
+      @prefixs = options[:prefixs] || []
+      @postfixs = options[:postfixs] || []
+
+      klass_to_headers(@klass)
+
+      Erubis::Eruby.new( File.read(export_template)).result(binding)
+    end
+
+    # Create an YAML template BAASED on an Excel spreadsheet for mapping headers
+    #
+    # * <tt>:title</tt> - Top level YAML node -defaults to ConfigGenerator.title
+    #
+    # * <tt>:model_as_dest</tt> - Override default treatment of using model as the SOURCE
+    #
+    # * <tt>:file</tt> - Write mappings direct to file name provided
+    #
+    def generate_from_excel(excel_file_name, options = {})
+
+      excel = Excel.new
+
+      puts "\n\n\nGenerating mapping from Excel file: #{excel_file_name}"
+
+      excel.open(excel_file_name)
+
+      sheet_number = options[:sheet_number] || 0
+
+      sheet = excel.worksheet( sheet_number )
+
+      @headers = excel.parse_headers(sheet, options[:header_row] || 0)
+
+      mappings = options[:title] || ConfigGenerator.title
+
+      if options[:model_as_dest]
+        headers.each_with_index { |s, i|  mappings += "       #srcs_column_heading_#{i}: #{s}\n" }
+      else
+        headers.each_with_index { |s, i|  mappings += "       #{s}: #dest_column_heading_#{i}\n" }
+      end
+
+      File.open(options[:file], 'w') { |f| f << mappings } if options[:file]
+
+      mappings
+
+    end
+
+  end
+
+end # DataShift

data/lib/generators/csv_generator.rb

@@ -1,92 +1,35 @@
-# Copyright:: (c) Autotelik Media Ltd 2011
+# Copyright:: (c) Autotelik Media Ltd 2015
 # Author ::   Tom Statter
-# Date ::     Aug 2011
 # License::   MIT
 #
-# Details::   Export a model to CSV
+# Details::   Export a model definition to CSV
 #
 #
-require 'generator_base'
-
 module DataShift
 
   class CsvGenerator < GeneratorBase
 
-    include DataShift::Logging
+    attr_accessor :file_name
 
-    def initialize(filename)
-      super(filename)
+    def initialize
+      super
     end
 
     # Create CSV file representing supplied Model
+    #
+    def generate(file_name, klass)
+      @file_name = file_name
 
-    def generate(klass, options = {})
-      @filename = options[:filename] if options[:filename]
-
-      prep_remove_list(options)
-
-      MethodDictionary.find_operators( klass )
-      @headers = MethodDictionary.assignments[klass]
-
-      @headers.delete_if{|h| @remove_list.include?( h.to_sym ) }
-
-      logger.info("CSVGenerator saving generated template #{@filename}")
-
-      CSV.open(@filename, "w") do |csv|
-        csv << @headers
-      end
-    end
-
-    def generate_with_associations(klass, options = {})
-      @filename = options[:filename] if options[:filename]
-
-      MethodDictionary.find_operators( klass )
-      MethodDictionary.build_method_details( klass )
-
-      work_list = MethodDetail::supported_types_enum.to_a - [ *options[:exclude] ]
-
-      prep_remove_list(options)
-
-      @headers = []
-
-      details_mgr = MethodDictionary.method_details_mgrs[klass]
-
-      work_list.each do |assoc_type|
-        method_details_for_assoc_type = details_mgr.get_list_of_method_details(assoc_type)
-
-        next if(method_details_for_assoc_type.nil? || method_details_for_assoc_type.empty?)
+      headers = Headers.klass_to_headers(klass)
 
-        method_details_for_assoc_type.each do |md|
-          comparable_association = md.operator.to_s.downcase.to_sym
+      logger.info("CSVGenerator saving generated Template #{@file_name}")
 
-          i = remove_list.index { |r| r == comparable_association }
+      csv_delim = DataShift::Exporters::Configuration.call.csv_delimiter
 
-          (i) ? remove_list.delete_at(i) : headers << "#{md.operator}"
-        end
+      CSV.open(file_name, 'w', col_sep: csv_delim ) do |csv|
+        csv << headers.sources
       end
 
-      logger.info("CSVGenerator saving generated with associations template #{@filename}")
-
-      CSV.open(@filename, "w") do |csv|
-        csv << @headers
-      end
-    end
-
-
-    # Create an CSV file representing supplied Model
-
-    def export(items, options = {})
-    end
-
-
-    private
-
-    # Take options and create a list of symbols to remove from headers
-    #
-    def prep_remove_list( options )
-      @remove_list = [ *options[:remove] ].compact.collect{|x| x.to_s.downcase.to_sym }
-
-      @remove_list += GeneratorBase::rails_columns if(options[:remove_rails])
     end
 
   end

data/lib/generators/excel_generator.rb

@@ -1,6 +1,5 @@
-# Copyright:: (c) Autotelik Media Ltd 2011
+# Copyright:: (c) Autotelik Media Ltd 2015
 # Author ::   Tom Statter
-# Date ::     Aug 2011
 # License::   MIT
 #
 # Details::   Export a model to Excel '97(-2007) file format.
@@ -8,130 +7,43 @@
 # TOD : Can we switch between .xls and XSSF (POI implementation of Excel 2007 OOXML (.xlsx) file format.)
 #
 #
-require 'generator_base'
 require 'excel'
-  
+
 module DataShift
-      
+
   class ExcelGenerator < GeneratorBase
 
-    include DataShift::Logging
-      
+    include DataShift::ExcelBase
+
     attr_accessor :excel
-  
-    def initialize(filename)
-      super(filename)
+    attr_accessor :file_name
+
+    def initialize
+      super
     end
 
     # Create an Excel file template (header row) representing supplied Model
-    # Options:
-    # * <tt>:filename</tt>
-    # * <tt>:autosize</tt> - Autosize all the columns
+    # file_name => Filename for generated template
     #
-    def generate(klass, options = {})
-     
-      prepare_excel(klass, options)
+    # See DataShift::Exporters::Configuration for options
+    #
+    def generate(file_name, klass, options = {})
 
-      prep_remove_list(options)  
-            
-      @headers = MethodDictionary.assignments[klass]
-      
-      @headers.delete_if{|h| @remove_list.include?( h.to_sym ) }
-      
-      @excel.set_headers( @headers )
+      @file_name = file_name
 
-      logger.info("ExcelGenerator saving generated template #{@filename}")
-        
-      #@excel.autosize if(options[:autosize])
-        
-      @excel.write( @filename )
-    end
+      start_excel(klass, options)
 
-      
-    # Create an Excel file template (header row) representing supplied Model
-    # and it's associations
-    # 
-    # Options:
-    # * <tt>:filename</tt>
-    # * <tt>:autosize</tt> - Autosize all the columns
-    #
-    # * <tt>:exclude</tt> - Association TYPE(s) to exclude.
-    #   You can specify a hash of {association_type => [array of association names] }
-    #   to exclude from the template.
-    #   
-    #     Possible association_type values are given by MethodDetail::supported_types_enum
-    #       ... [:assignment, :belongs_to, :has_one, :has_many]
-    #       
-    # * <tt>:remove</tt> - Association NAME(s) to remove .. :title, :id, :name
-    # .
-    # * <tt>:remove_rails</tt> - Remove Rails DB columns :
-    #           :id, :created_at, :created_on, :updated_at, :updated_on
-    #   
-    def generate_with_associations(klass, options = {})
+      @headers = Headers.klass_to_headers(klass)
+
+      @excel.set_headers(@headers)
 
-      prepare_excel(klass, options)
-         
-      MethodDictionary.build_method_details( klass )
-           
-      work_list = MethodDetail::supported_types_enum.to_a - [ *options[:exclude] ]
-        
-      prep_remove_list(options)    
-      
-      @headers = []
-      
-      details_mgr = MethodDictionary.method_details_mgrs[klass]
-                  
-      work_list.each do |assoc_type|
-        method_details_for_assoc_type = details_mgr.get_list_of_method_details(assoc_type)
-          
-        next if(method_details_for_assoc_type.nil? || method_details_for_assoc_type.empty?)
-        
-        method_details_for_assoc_type.each do |md| 
-          comparable_association = md.operator.to_s.downcase.to_sym
+      logger.info("ExcelGenerator saving generated template #{@file_name}")
 
-          i = remove_list.index { |r| r == comparable_association }
-             
-          (i) ? remove_list.delete_at(i) : headers << "#{md.operator}"
-        end
-      end
-        
-      @excel.set_headers( headers )
-        
       # @excel.autosize if(options[:autosize])
-                
-      @excel.write( filename() )
-    end
-      
-    private
-      
-    # Take options and create a list of symbols to remove from headers
-    # 
-    def prep_remove_list( options )
-      @remove_list = [ *options[:remove] ].compact.collect{|x| x.to_s.downcase.to_sym }
-         
-      @remove_list += GeneratorBase::rails_columns if(options[:remove_rails])
-    end
-    
-    
-    def prepare_excel(klass, options = {})
-      @filename = options[:filename] if  options[:filename]
-        
-      logger.info("ExcelGenerator creating template with associations for class #{klass}")
-        
-      @excel = Excel.new()
 
-      name  = options[:sheet_name] || klass.name
-        
-      sheet = @excel.create_worksheet( :name => name ) 
-    
-      unless sheet
-        logger.error("Excel failed to create WorkSheet for #{name}")
-        
-        raise "Failed to create Excel WorkSheet for #{name}" 
-      end
-        
-      MethodDictionary.find_operators( klass )
+      @excel.write( @file_name )
     end
+
   end # ExcelGenerator
-  
-end # DataShift
+
+end # DataShift