guara 0.0.1.rc

data/lib/active_migration/migration.rb

@@ -0,0 +1,211 @@
+
+require 'spreadsheet'
+require 'active_support'
+
+module ActiveMigration
+  class ActiveMigartionError < StandardError; end
+  class ActiveMigartionDataSourceError < ActiveMigartionError; end  
+  class AciteMigrationInvalidRecordError < ActiveMigartionError; end  
+  
+  
+  class Schema
+    ##
+    # @attr columns Hash with name and type
+    # @attr format Describe data format
+    # @attr url define path or object schema
+    attr_accessor :columns, :format, :url
+    
+  end
+  
+  ##
+  # Module Data Transformation
+  # When passing schema file or struct, the data between datasources
+  # needs a transformations 
+  module Transformer
+    ##
+    # called on start of migration
+    def begin(schema_from, schema_to)
+      # nothing      
+    end
+    
+    ##
+    # called on start of transaction
+    def begin_transaction(schema_from, schema_to)
+      # nothing
+    end
+    
+    ##
+    # transform from data row to destinate data row 
+    # @result true, false or :ignore to ignore this row
+    def transform(row)
+      raise "Implements transform method!"
+    end
+    
+    ##
+    # called on ending migration
+    def end(schema_from, schema_to)
+      # nothing
+    end
+    
+    ##
+    # called on ending transaction
+    def end_transaction(schema_from, schema_to)
+      # nothing
+    end
+    
+    ##
+    #
+    def after_row_saved(row, object)
+      #
+    end
+    
+    def transform_ignore_fields(row)
+      #delete ignore
+      row.reject! { |key,value| key.to_s.start_with?("ignore") }
+    end
+    
+  end  
+  
+  
+  ##
+  # == Class Migration
+  # 
+  # Migrate data between  data source and transforme to destination 
+  #
+  class Migration
+    attr_accessor :schema_from, :schema_to, :transformer, :name
+  
+    # constructor
+    def initialize(schema_url=nil)
+      self.load_schema(schema_url) unless schema_url.nil?
+    end
+    
+    ##
+    # load yml schemas from and to
+    def load_schema(url)
+      schema = YAML::load(File.open(url))
+      self.schema_from = schema[:from]
+      self.schema_to = schema[:to]
+    end
+    
+    
+    ##
+    # Running migration from configured files
+    # 
+    # ps> Default Behaviour Ignore First line - assumes head line
+    
+    def migrate!
+    
+      raise "schema_from needs" if @schema_from.nil?
+      raise "schema_to needs" if @schema_to.nil?
+      
+      res = @transformer.begin_transaction(@schema_from, @schema_to) unless @transformer.nil?
+      
+      ActiveRecord::Base.transaction do      
+        begin_migration()
+    
+        # TODO: Make flexible configurable and more input formats
+        if @schema_from[:format].to_s.to_sym == :XLS
+            xls_migrate()
+        end
+      
+        end_migration()
+      end
+      
+      res = @transformer.end_transaction(@schema_from, @schema_to) unless @transformer.nil?
+      
+      return true
+    end
+    
+    
+    def xls_migrate
+      begin
+        @xls = Spreadsheet.open @schema_from[:url]
+        # TODO: make others workbook accessible by configuration
+        sheet = @xls.worksheet 0
+    
+        @line = 0
+    
+        # ignore head line
+        sheet.each 1 do |row|
+          @column = 0
+          row_to = { }
+          
+          #read schema columns and types
+          @schema_from[:columns].each do |schema_column, schema_type|
+            row_to.merge!(schema_column.to_sym => row[@column])
+            @column+=1
+          end
+          
+          #transform row to @schema_to
+          res = true
+          res = @transformer.transform(row_to) unless @transformer.nil?
+          
+          if (res!=:ignore)
+            res = res==true && send_row_to_schema(row_to)
+            raise_migration if (res==false)
+            
+            @transformer.after_row_saved(row_to, @last_object) unless @transformer.nil?
+          end
+          
+          @line+=1
+        end
+      rescue Exception => e
+        line = @line.nil? ? 0 : @line
+        column = @column.nil? ? 0 : @column
+        raise ActiveMigartionDataSourceError.new ("Failing import excel source format from %s. %d:%d [ignored head]. " % [@schema_from[:url], column, line]).concat(e.message).concat("\n----"+e.backtrace.to_yaml)
+      end
+    end
+    
+    
+    def begin_migration
+      # TODO: make transactional
+      res = @transformer.begin(@schema_from, @schema_to) unless @transformer.nil?
+    end
+    
+    
+    def end_migration
+      res = @transformer.end(@schema_from, @schema_to) unless @transformer.nil?
+    end
+    
+    def raise_migration
+      raise "failing migration %s.  Line: %d, Column: %d" % [@name, @line, @column]
+    end
+  
+    def send_row_to_schema(row)
+      if @schema_to[:format].to_sym == :ACTIVE_RECORD
+      
+        # TODO: optimize on initialize migration
+        class_schema_to = eval @schema_to[:url]
+      
+        @last_object = class_schema_to.new(row)
+        res = @last_object.save
+      
+        if (!res)
+          raise AciteMigrationInvalidRecordError.new "[Schema:%s] Error on send to ACTIVE_RECORD %s. \n%s \nrow: \n%s" % [@name, @schema_to[:url], @last_object.errors.to_yaml, row.to_yaml]
+        end
+      
+        return res
+      
+      else
+        raise "Not valid schema::TO format! %s" % @name  
+      end
+    end
+    
+    ##
+    # loads yml file and convert to hash on schema_from
+    # @deprecated
+    def load_schema_from(url)
+      self.schema_from = YAML::load(File.open(url))
+    end
+    
+    ##
+    # load yml file and convert to hash on schema_to
+    # @deprecated
+    def load_schema_to(url)
+      self.schema_to = YAML::load(File.open(url))      
+    end
+    
+  end
+  
+end

data/lib/active_migration/schemas.rb

@@ -0,0 +1,143 @@
+
+module ActiveMigration
+  
+  module Schemas
+    class ActiveMigartionSchemasError < StandardError; end
+    
+    ##
+    # Façade class
+    # make easy way to running batchs schemas migrations
+    #
+    # === File Format
+    # name_migration:  
+    #   :type: CUSTOM, SCHEMA 
+    #   from: URL to from schema or Custom Param
+    #   to: URL to destinate schema or Custom Param
+    #   transformer: class will be called to data transformations
+    class SchemasMigration
+    
+      attr_accessor :transformer, :schemas
+    
+      def initialize(file_schemas)
+        raise "Invalid File: should not null!" if file_schemas.nil?
+        raise "Invalid File: should not exists!" if not File.exists?(file_schemas)
+        @schemas = YAML::load(File.open(file_schemas))
+      end
+      
+      def migrate!
+        ActiveRecord::Base.transaction do   
+          @schemas.each do |key,schema|
+            
+            @migration_name = key
+            @schema = schema
+            
+            msg = "Starting external migration: %s..." % @migration_name
+            Rails.logger.info msg
+            puts msg
+            
+            result = run_migration_job
+            
+            raise ActiveMigartionSchemasError.new("Failing Migrate Schemas: %s" % key) if not result
+            
+            msg = "Ending: %s." % key
+            Rails.logger.info msg
+            puts msg
+          end
+        end
+      end
+    
+
+      def eval_class(class_str)
+        begin
+          class_found = eval class_str
+          raise "its %s not a class" % class_str if !class_found.is_a?(Class) 
+        rescue
+          class_found = false
+        end
+      end
+
+      def transformer_class=(class_str)
+        file_name = class_str.underscore + ".rb"
+      
+        class_found = eval_class(class_str)
+      
+        if class_found==false
+          require_dependence(file_name)
+          class_found = eval_class(class_str)
+          raise "[%s] Invald informed Transformer: %s.  Schema: %s" % [@migration_name, class_str, @schema.to_yaml] if class_found == false 
+        end
+      
+        @transformer = (eval class_str).new @schema
+      end
+    
+      def require_dependence(file_name)
+      
+        Rails.logger.warn "Requiring file %s" % file_name
+      
+        search_dependency(file_name).each do |file|
+          if File.exists? file
+            Rails.logger.debug "Including file %s" % file
+            require file
+            break
+          end
+        end
+      end
+    
+      ##
+      # @return Array with possible location of file
+      def search_dependency(file_name)
+        files = []
+        files << Rails.root.join("db/external_migrate/" + file_name)
+      
+        #possibility paths
+        unless File.exists?(files[0])
+          Dir[file_name,
+              File.expand_path("**/external_migrate/**/" + file_name),
+              "../" + file_name,
+              "../../" + file_name,
+              File.expand_path("**/" + file_name)].each { |f| files << f }
+        end
+      
+        files
+      end
+    
+      def run_migration_job
+        transformer_from_schema
+        
+        case @schema[:type]
+          when :SCHEMA
+            self.migrate_schema
+          when :CUSTOM
+            self.migrate_custom
+        end
+      end
+    
+      def migrate_schema
+        migration = ActiveMigration::Migration.new
+        migration.name = @migration_name
+        migration.schema_from = @schema[:from]
+        migration.schema_to   = @schema[:to]
+        migration.transformer = @transformer if not @transformer.nil?
+        #rock!
+        migration.migrate!
+      end
+    
+      def migrate_custom
+        raise "Transformer not assigned" if @transformer.nil?
+        raise "Invalid Custom Migration Transformer" if not @transformer.respond_to?(:migrate!) 
+      
+        @transformer.migrate!
+      end
+    
+      def transformer_from_schema
+        if @schema.include? :transformer
+          self.transformer_class = @schema[:transformer]
+        else
+          @transformer = nil
+        end
+      end
+    
+    
+    end
+  end
+end

data/lib/active_migration/spelling_fix.rb

@@ -0,0 +1,20 @@
+
+module ActiveMigration
+  class SpellingFix
+    
+    def initialize(url)
+      @rules = YAML::load(File.open(url))
+    end
+    
+    def fix!(term)
+      return term if term.nil? || term.empty?
+      @rules.each do |before,after|
+        term.gsub! before, after
+      end
+      
+      term
+    end
+    
+    
+  end
+end

data/lib/active_migration/transformer/grouped_field_fixed_spelling.rb

@@ -0,0 +1,110 @@
+
+module ActiveMigration
+  module Transformer
+    class GroupedFieldFixedSpelling
+      
+      include ActiveMigration::Transformer
+      include ApplicationHelper
+      
+      attr_accessor :domain_name, :domain_dest
+      
+      def initialize(schema)
+        @schema            = schema
+        @field_group       = :name
+        @field_group_fixed = :name_correct
+        
+        @group  = {}
+        @rows   = []
+        
+        @domain_name = "domain"
+      end
+      
+      def begin(schema_from, schema_to)
+        @schema_from = schema_from
+        @schema_to = schema_to
+        @group = {}
+        @domain_dictionary = {}
+        @rows = []
+      end
+      
+      def transform(row)
+        begin
+          #cache
+          @rows << row
+          
+          fix_field row
+          
+          return :ignore if row[@field_group].nil?
+          
+          #validating and cleanup
+          row[@field_group].gsub! /\s{2,}/, ' '
+          row.delete @field_group_fixed
+          
+          @group[row[@field_group]] = row
+          
+          return :ignore
+        rescue Exception => e  
+          Rails.logger.debug "Error on transform. "+e.message+"\n"+e.backtrace.join("\n ")
+          false
+        end
+      end
+      
+      def fix_field(row)
+        #has name_correct?
+        if (!row[@field_group_fixed].nil? && !row[@field_group_fixed].to_s.empty?)
+          row[@field_group_fixed]               = row[@field_group_fixed].to_s.strip
+          @domain_dictionary[row[@field_group]] = row[@field_group_fixed]
+          row[@field_group]                     = row[@field_group_fixed]
+        end
+      end
+    
+      def end(schema_from, schema_to)
+        save_on_cache
+        save_on_db
+      end
+    
+      def schema_to_class
+        eval @schema_to[:url]
+      end
+    
+      def save_on_db
+    
+        class_to = schema_to_class
+    
+        @group.each do |key, row|
+          if not class_to.send(("find_by_%s" % @field_group).to_sym, row[@field_group])
+            record = class_to.new(row)
+            puts "SaveOnDB: "+row.to_yaml
+            record.save!
+          end
+        end
+      end
+  
+      def save_on_cache
+    
+        #cached path
+        @cache_path = "db/external_migrate/cache/" 
+        Dir.mkdir path if not Dir.exists? @cache_path
+    
+        puts_on_file @cache_path + "#{@domain_name}_dictionary.yml", @domain_dictionary.to_yaml
+        #puts_on_file @cache_path + "#{@domain_name}_raw.csv", @rows.map { |i| i.values.join(",")}.join("\n")
+        puts_on_file @cache_path + "#{@domain_name}_grouped.csv" do
+          lines = []
+          @group.each do |key, data|
+            line = []
+            data.each do |key, value|
+              if value.is_a? String
+                line << value
+              elsif value.respond_to? :name
+                line << value.name
+              end
+            end
+            lines << line.join(",")
+          end
+          lines.join("\n")
+        end
+        #puts_on_file path + "#{@domain_name}.xml", @rows.to_xml
+      end
+    end
+  end
+end