active_extend 0.0.1 → 0.0.2

data/lib/active_extend/active_disablable.rb

@@ -25,7 +25,7 @@ module ActiveExtend
     def disable
       self.enabled = false
     end
-  
+    
     def enable
       self.enabled = true
     end

data/lib/active_extend/railtie.rb

@@ -0,0 +1,10 @@
+require 'active_extend'
+require 'rails'
+
+module ActiveExtend
+  class Railtie < Rails::Railtie
+    rake_tasks do
+      load File.expand_path('../../tasks/external_migrate.rake', __FILE__)
+    end
+  end
+end

data/lib/active_extend/version.rb

@@ -1,3 +1,3 @@
 module ActiveExtend
-  VERSION = "0.0.1".freeze
+  VERSION = "0.0.2".freeze
 end

data/lib/active_extend.rb

@@ -1,5 +1,7 @@
 
 require "active_extend/active_disablable"
+require "active_extend/railtie" if defined?(Rails)
+require "active_extend/string_helper"
 
 module ActiveExtend
 end

data/lib/external_migration/migration.rb

@@ -1,5 +1,4 @@
 
-require 'spreadsheet'
 require 'active_support'
 
 module ExternalMigration
@@ -25,7 +24,8 @@ module ExternalMigration
     ##
     # called on start of migration
     def begin(schema_from, schema_to)
-      # nothing      
+      @schema_from = schema_from
+      @schema_to = schema_to
     end
     
     ##
@@ -66,6 +66,10 @@ module ExternalMigration
     
   end  
   
+  module Decoder
+    def migrate!
+    end
+  end
   
   ##
   # == Class Migration
@@ -101,12 +105,18 @@ module ExternalMigration
       
       res = @transformer.begin_transaction(@schema_from, @schema_to) unless @transformer.nil?
       
+      @line = 0
+      
       ActiveRecord::Base.transaction do      
         begin_migration()
     
         # TODO: Make flexible configurable and more input formats
         if @schema_from[:format].to_s.to_sym == :XLS
-            xls_migrate()
+          xls_migrate()
+        elsif @schema_from[:format].to_s.to_sym == :TXT_FIXED
+          decoder = ExternalMigration::TextFixed.new(@schema_from)
+          decoder.migration = self
+          decoder.migrate!
         end
       
         end_migration()
@@ -117,6 +127,34 @@ module ExternalMigration
       return true
     end
     
+    def migreate_row!(row_to)
+      @line += 1
+      begin
+        #transform row to @schema_to
+        res = true
+        res = @transformer.transform(row_to) unless @transformer.nil?
+        
+        if (res!=:ignore)
+          res = res==true && send_row_to_schema(row_to)
+          raise_migration if (res==false)
+          
+          @transformer.after_row_saved(row_to, @last_object) unless @transformer.nil?
+        end
+      rescue Exception => e
+        line = @line.nil? ? 0 : @line
+        column = @column.nil? ? 0 : @column
+        
+        obj = @last_object || (e.respond_to?(:record) && (e.record)) || nil
+        
+        if !obj.nil?
+          raise ActiveMigartionDataSourceError.new obj.errors.to_yaml.to_s.concat("Failing import excel source format from %s. %d:%d [ignored head]. " % [@schema_from[:url], column, line]).concat(e.message).concat("\n----"+e.backtrace.to_yaml)         
+        else
+          raise ActiveMigartionDataSourceError.new ("Failing import excel source format from %s. %d:%d [ignored head]. " % [@schema_from[:url], column, line]).concat(e.message).concat("\n----"+e.backtrace.to_yaml)
+        end
+      end
+
+    end
+    
     
     def xls_migrate
       begin
@@ -124,8 +162,6 @@ module ExternalMigration
         # TODO: make others workbook accessible by configuration
         sheet = @xls.worksheet 0
     
-        @line = 0
-    
         # ignore head line
         sheet.each 1 do |row|
           @column = 0
@@ -137,23 +173,19 @@ module ExternalMigration
             @column+=1
           end
           
-          #transform row to @schema_to
-          res = true
-          res = @transformer.transform(row_to) unless @transformer.nil?
-          
-          if (res!=:ignore)
-            res = res==true && send_row_to_schema(row_to)
-            raise_migration if (res==false)
-            
-            @transformer.after_row_saved(row_to, @last_object) unless @transformer.nil?
-          end
-          
-          @line+=1
+          self.migrate_row! row_to
         end
       rescue Exception => e
         line = @line.nil? ? 0 : @line
         column = @column.nil? ? 0 : @column
-        raise ActiveMigartionDataSourceError.new ("Failing import excel source format from %s. %d:%d [ignored head]. " % [@schema_from[:url], column, line]).concat(e.message).concat("\n----"+e.backtrace.to_yaml)
+        
+        obj = @last_object || (e.respond_to?(:record) && (e.record)) || nil
+        
+        if !obj.nil?
+          raise ActiveMigartionDataSourceError.new obj.errors.to_yaml.to_s.concat("Failing import excel source format from %s. %d:%d [ignored head]. " % [@schema_from[:url], column, line]).concat(e.message).concat("\n----"+e.backtrace.to_yaml)         
+        else
+          raise ActiveMigartionDataSourceError.new ("Failing import excel source format from %s. %d:%d [ignored head]. " % [@schema_from[:url], column, line]).concat(e.message).concat("\n----"+e.backtrace.to_yaml)
+        end
       end
     end
     
@@ -173,16 +205,19 @@ module ExternalMigration
     end
   
     def send_row_to_schema(row)
-      if @schema_to[:format].to_sym == :ACTIVE_RECORD
       
+      if @schema_to[:format].to_sym == :ACTIVE_RECORD
+        
         # TODO: optimize on initialize migration
         class_schema_to = eval @schema_to[:url]
-      
+        
         @last_object = class_schema_to.new(row)
         res = @last_object.save
       
         if (!res)
-          raise AciteMigrationInvalidRecordError.new "[Schema:%s] Error on send to ACTIVE_RECORD %s. \n%s \nrow: \n%s" % [@name, @schema_to[:url], @last_object.errors.to_yaml, row.to_yaml]
+          msg = "[Schema:%s] Error on send to ACTIVE_RECORD %s. \n%s \nrow: \n%s" % [@name, @schema_to[:url], @last_object.errors.to_yaml, row.to_yaml]
+          Rails.logger.error msg
+          raise AciteMigrationInvalidRecordError.new msg
         end
       
         return res

data/lib/external_migration/schemas.rb

@@ -39,13 +39,23 @@ module ExternalMigration
             
             raise ActiveMigartionSchemasError.new("Failing Migrate Schemas: %s" % key) if not result
             
-            msg = "Ending: %s." % key
+            msg = "Ending: %s." % @migration_name
             Rails.logger.info msg
             puts msg
           end
         end
       end
     
+        def run_migration_job
+          transformer_from_schema()
+
+          case @schema[:type]
+            when :SCHEMA
+              self.migrate_schema
+            when :CUSTOM
+              self.migrate_custom
+          end
+        end
 
       def eval_class(class_str)
         begin
@@ -57,7 +67,12 @@ module ExternalMigration
       end
 
       def transformer_class=(class_str)
-        file_name = class_str.underscore + ".rb"
+        
+        path = class_str.split("::")
+        
+        
+        path.map!(&:underscore)
+        file_name = path.join("/") + ".rb"
       
         class_found = eval_class(class_str)
       
@@ -101,17 +116,6 @@ module ExternalMigration
         files
       end
     
-      def run_migration_job
-        transformer_from_schema
-        
-        case @schema[:type]
-          when :SCHEMA
-            self.migrate_schema
-          when :CUSTOM
-            self.migrate_custom
-        end
-      end
-    
       def migrate_schema
         migration = ExternalMigration::Migration.new
         migration.name = @migration_name

data/lib/external_migration/text_fixed.rb

@@ -0,0 +1,53 @@
+
+
+module ExternalMigration
+  
+  #
+  # Schema format:
+  #
+  # format: TXT_FIXED
+  # url: text to read
+  # ignore_lines: default 1
+  # encoding: default 'windows-1251:utf-8'
+  class TextFixed
+    include ExternalMigration::Decoder
+    
+    attr_accessor :migration, :ignore_lines
+    
+    def initialize(schema)
+      self.schema = schema
+      @ignore_lines = schema[:ignore_lines] || 1
+      @enconding = schema[:encoding] || 'windows-1251:utf-8'
+    end
+    
+    def schema=(schema)
+      @schema = schema
+    end
+    
+    def migrate!
+      puts "opening file #{@schema[:url]}"
+      file = File.open(Rails.root.join(@schema[:url]), "r:#{@enconding}")
+      file.each_line do |line|
+        if @ignore_lines>0
+          @ignore_lines -= 1
+          next line
+        end
+        
+        row = {}
+        line_index = 0
+        @schema[:columns].each do |column, column_prop|
+          row[column.to_sym] = line[line_index..(line_index+column_prop[:length]-1)]
+          row[column.to_sym].strip! if @schema[:strip_columns] == :true && !row[column.to_sym].nil?
+          line_index += column_prop[:length]
+        end
+        row.keys.each {|k| row.delete k if k.to_s =~ /ignore\d+/ }
+        puts row.to_yaml
+        @migration.migreate_row! row
+      end
+      file.close
+      
+    end
+    
+  end
+  
+end

data/lib/external_migration/transformer/grouped_field_fixed_spelling.rb

@@ -1,9 +1,9 @@
 
-module ActiveMigration
+module ExternalMigration
   module Transformer
     class GroupedFieldFixedSpelling
       
-      include ActiveMigration::Transformer
+      include ExternalMigration::Transformer
       include ApplicationHelper
       
       attr_accessor :domain_name, :domain_dest
@@ -84,7 +84,7 @@ module ActiveMigration
     
         #cached path
         @cache_path = "db/external_migrate/cache/" 
-        Dir.mkdir path if not Dir.exists? @cache_path
+        Dir.mkdir @cache_path if not Dir.exists? @cache_path
     
         puts_on_file @cache_path + "#{@domain_name}_dictionary.yml", @domain_dictionary.to_yaml
         #puts_on_file @cache_path + "#{@domain_name}_raw.csv", @rows.map { |i| i.values.join(",")}.join("\n")

data/lib/external_migration.rb

@@ -1,4 +1,6 @@
 
+require 'spreadsheet'
+
 #main
 require "external_migration/migration"
 
@@ -6,5 +8,8 @@ require "external_migration/migration"
 require "external_migration/dictionary"
 require "external_migration/spelling_fix"
 
+#decoders
+require "external_migration/text_fixed"
+
 #schemas
 require "external_migration/schemas"

data/lib/tasks/external_migrate.rake

@@ -0,0 +1,12 @@
+namespace :db do
+  task external_migrate: :environment do    
+    require "external_migration"
+    
+    Dir[Rails.root.join("db/external_migrate/*schemas.yml")].each do |file|
+      @migration_schemas = ExternalMigration::Schemas::SchemasMigration.new file
+      if @migration_schemas.migrate!
+        Rails.logger.info "Migration success! %s" % file
+      end
+    end
+  end
+end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: active_extend
 version: !ruby/object:Gem::Version
-  version: 0.0.1
+  version: 0.0.2
   prerelease: 
 platform: ruby
 authors:
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2012-12-24 00:00:00.000000000 Z
+date: 2013-04-29 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rails
@@ -28,14 +28,14 @@ dependencies:
       - !ruby/object:Gem::Version
         version: 3.2.9
 - !ruby/object:Gem::Dependency
-  name: sqlite3
+  name: spreadsheet
   requirement: !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
       - !ruby/object:Gem::Version
         version: '0'
-  type: :development
+  type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     none: false
@@ -51,6 +51,7 @@ extensions: []
 extra_rdoc_files: []
 files:
 - lib/active_extend/active_disablable.rb
+- lib/active_extend/railtie.rb
 - lib/active_extend/string_helper.rb
 - lib/active_extend/version.rb
 - lib/active_extend.rb
@@ -59,9 +60,10 @@ files:
 - lib/external_migration/migration.rb
 - lib/external_migration/schemas.rb
 - lib/external_migration/spelling_fix.rb
+- lib/external_migration/text_fixed.rb
 - lib/external_migration/transformer/grouped_field_fixed_spelling.rb
 - lib/external_migration.rb
-- lib/tasks/active_extends_tasks.rake
+- lib/tasks/external_migrate.rake
 - MIT-LICENSE
 - Rakefile
 - README.rdoc

data/lib/tasks/active_extends_tasks.rake

@@ -1,4 +0,0 @@
-# desc "Explaining what the task does"
-# task :active_extend do
-#   # Task goes here
-# end