active_extend 0.0.1 → 0.0.2

Sign up to get free protection for your applications and to get access to all the features.
@@ -25,7 +25,7 @@ module ActiveExtend
25
25
  def disable
26
26
  self.enabled = false
27
27
  end
28
-
28
+
29
29
  def enable
30
30
  self.enabled = true
31
31
  end
@@ -0,0 +1,10 @@
1
+ require 'active_extend'
2
+ require 'rails'
3
+
4
+ module ActiveExtend
5
+ class Railtie < Rails::Railtie
6
+ rake_tasks do
7
+ load File.expand_path('../../tasks/external_migrate.rake', __FILE__)
8
+ end
9
+ end
10
+ end
@@ -1,3 +1,3 @@
1
1
  module ActiveExtend
2
- VERSION = "0.0.1".freeze
2
+ VERSION = "0.0.2".freeze
3
3
  end
data/lib/active_extend.rb CHANGED
@@ -1,5 +1,7 @@
1
1
 
2
2
  require "active_extend/active_disablable"
3
+ require "active_extend/railtie" if defined?(Rails)
4
+ require "active_extend/string_helper"
3
5
 
4
6
  module ActiveExtend
5
7
  end
@@ -1,5 +1,4 @@
1
1
 
2
- require 'spreadsheet'
3
2
  require 'active_support'
4
3
 
5
4
  module ExternalMigration
@@ -25,7 +24,8 @@ module ExternalMigration
25
24
  ##
26
25
  # called on start of migration
27
26
  def begin(schema_from, schema_to)
28
- # nothing
27
+ @schema_from = schema_from
28
+ @schema_to = schema_to
29
29
  end
30
30
 
31
31
  ##
@@ -66,6 +66,10 @@ module ExternalMigration
66
66
 
67
67
  end
68
68
 
69
+ module Decoder
70
+ def migrate!
71
+ end
72
+ end
69
73
 
70
74
  ##
71
75
  # == Class Migration
@@ -101,12 +105,18 @@ module ExternalMigration
101
105
 
102
106
  res = @transformer.begin_transaction(@schema_from, @schema_to) unless @transformer.nil?
103
107
 
108
+ @line = 0
109
+
104
110
  ActiveRecord::Base.transaction do
105
111
  begin_migration()
106
112
 
107
113
  # TODO: Make flexible configurable and more input formats
108
114
  if @schema_from[:format].to_s.to_sym == :XLS
109
- xls_migrate()
115
+ xls_migrate()
116
+ elsif @schema_from[:format].to_s.to_sym == :TXT_FIXED
117
+ decoder = ExternalMigration::TextFixed.new(@schema_from)
118
+ decoder.migration = self
119
+ decoder.migrate!
110
120
  end
111
121
 
112
122
  end_migration()
@@ -117,6 +127,34 @@ module ExternalMigration
117
127
  return true
118
128
  end
119
129
 
130
+ def migreate_row!(row_to)
131
+ @line += 1
132
+ begin
133
+ #transform row to @schema_to
134
+ res = true
135
+ res = @transformer.transform(row_to) unless @transformer.nil?
136
+
137
+ if (res!=:ignore)
138
+ res = res==true && send_row_to_schema(row_to)
139
+ raise_migration if (res==false)
140
+
141
+ @transformer.after_row_saved(row_to, @last_object) unless @transformer.nil?
142
+ end
143
+ rescue Exception => e
144
+ line = @line.nil? ? 0 : @line
145
+ column = @column.nil? ? 0 : @column
146
+
147
+ obj = @last_object || (e.respond_to?(:record) && (e.record)) || nil
148
+
149
+ if !obj.nil?
150
+ raise ActiveMigartionDataSourceError.new obj.errors.to_yaml.to_s.concat("Failing import excel source format from %s. %d:%d [ignored head]. " % [@schema_from[:url], column, line]).concat(e.message).concat("\n----"+e.backtrace.to_yaml)
151
+ else
152
+ raise ActiveMigartionDataSourceError.new ("Failing import excel source format from %s. %d:%d [ignored head]. " % [@schema_from[:url], column, line]).concat(e.message).concat("\n----"+e.backtrace.to_yaml)
153
+ end
154
+ end
155
+
156
+ end
157
+
120
158
 
121
159
  def xls_migrate
122
160
  begin
@@ -124,8 +162,6 @@ module ExternalMigration
124
162
  # TODO: make others workbook accessible by configuration
125
163
  sheet = @xls.worksheet 0
126
164
 
127
- @line = 0
128
-
129
165
  # ignore head line
130
166
  sheet.each 1 do |row|
131
167
  @column = 0
@@ -137,23 +173,19 @@ module ExternalMigration
137
173
  @column+=1
138
174
  end
139
175
 
140
- #transform row to @schema_to
141
- res = true
142
- res = @transformer.transform(row_to) unless @transformer.nil?
143
-
144
- if (res!=:ignore)
145
- res = res==true && send_row_to_schema(row_to)
146
- raise_migration if (res==false)
147
-
148
- @transformer.after_row_saved(row_to, @last_object) unless @transformer.nil?
149
- end
150
-
151
- @line+=1
176
+ self.migrate_row! row_to
152
177
  end
153
178
  rescue Exception => e
154
179
  line = @line.nil? ? 0 : @line
155
180
  column = @column.nil? ? 0 : @column
156
- raise ActiveMigartionDataSourceError.new ("Failing import excel source format from %s. %d:%d [ignored head]. " % [@schema_from[:url], column, line]).concat(e.message).concat("\n----"+e.backtrace.to_yaml)
181
+
182
+ obj = @last_object || (e.respond_to?(:record) && (e.record)) || nil
183
+
184
+ if !obj.nil?
185
+ raise ActiveMigartionDataSourceError.new obj.errors.to_yaml.to_s.concat("Failing import excel source format from %s. %d:%d [ignored head]. " % [@schema_from[:url], column, line]).concat(e.message).concat("\n----"+e.backtrace.to_yaml)
186
+ else
187
+ raise ActiveMigartionDataSourceError.new ("Failing import excel source format from %s. %d:%d [ignored head]. " % [@schema_from[:url], column, line]).concat(e.message).concat("\n----"+e.backtrace.to_yaml)
188
+ end
157
189
  end
158
190
  end
159
191
 
@@ -173,16 +205,19 @@ module ExternalMigration
173
205
  end
174
206
 
175
207
  def send_row_to_schema(row)
176
- if @schema_to[:format].to_sym == :ACTIVE_RECORD
177
208
 
209
+ if @schema_to[:format].to_sym == :ACTIVE_RECORD
210
+
178
211
  # TODO: optimize on initialize migration
179
212
  class_schema_to = eval @schema_to[:url]
180
-
213
+
181
214
  @last_object = class_schema_to.new(row)
182
215
  res = @last_object.save
183
216
 
184
217
  if (!res)
185
- raise AciteMigrationInvalidRecordError.new "[Schema:%s] Error on send to ACTIVE_RECORD %s. \n%s \nrow: \n%s" % [@name, @schema_to[:url], @last_object.errors.to_yaml, row.to_yaml]
218
+ msg = "[Schema:%s] Error on send to ACTIVE_RECORD %s. \n%s \nrow: \n%s" % [@name, @schema_to[:url], @last_object.errors.to_yaml, row.to_yaml]
219
+ Rails.logger.error msg
220
+ raise AciteMigrationInvalidRecordError.new msg
186
221
  end
187
222
 
188
223
  return res
@@ -39,13 +39,23 @@ module ExternalMigration
39
39
 
40
40
  raise ActiveMigartionSchemasError.new("Failing Migrate Schemas: %s" % key) if not result
41
41
 
42
- msg = "Ending: %s." % key
42
+ msg = "Ending: %s." % @migration_name
43
43
  Rails.logger.info msg
44
44
  puts msg
45
45
  end
46
46
  end
47
47
  end
48
48
 
49
+ def run_migration_job
50
+ transformer_from_schema()
51
+
52
+ case @schema[:type]
53
+ when :SCHEMA
54
+ self.migrate_schema
55
+ when :CUSTOM
56
+ self.migrate_custom
57
+ end
58
+ end
49
59
 
50
60
  def eval_class(class_str)
51
61
  begin
@@ -57,7 +67,12 @@ module ExternalMigration
57
67
  end
58
68
 
59
69
  def transformer_class=(class_str)
60
- file_name = class_str.underscore + ".rb"
70
+
71
+ path = class_str.split("::")
72
+
73
+
74
+ path.map!(&:underscore)
75
+ file_name = path.join("/") + ".rb"
61
76
 
62
77
  class_found = eval_class(class_str)
63
78
 
@@ -101,17 +116,6 @@ module ExternalMigration
101
116
  files
102
117
  end
103
118
 
104
- def run_migration_job
105
- transformer_from_schema
106
-
107
- case @schema[:type]
108
- when :SCHEMA
109
- self.migrate_schema
110
- when :CUSTOM
111
- self.migrate_custom
112
- end
113
- end
114
-
115
119
  def migrate_schema
116
120
  migration = ExternalMigration::Migration.new
117
121
  migration.name = @migration_name
@@ -0,0 +1,53 @@
1
+
2
+
3
+ module ExternalMigration
4
+
5
+ #
6
+ # Schema format:
7
+ #
8
+ # format: TXT_FIXED
9
+ # url: text to read
10
+ # ignore_lines: default 1
11
+ # encoding: default 'windows-1251:utf-8'
12
+ class TextFixed
13
+ include ExternalMigration::Decoder
14
+
15
+ attr_accessor :migration, :ignore_lines
16
+
17
+ def initialize(schema)
18
+ self.schema = schema
19
+ @ignore_lines = schema[:ignore_lines] || 1
20
+ @enconding = schema[:encoding] || 'windows-1251:utf-8'
21
+ end
22
+
23
+ def schema=(schema)
24
+ @schema = schema
25
+ end
26
+
27
+ def migrate!
28
+ puts "opening file #{@schema[:url]}"
29
+ file = File.open(Rails.root.join(@schema[:url]), "r:#{@enconding}")
30
+ file.each_line do |line|
31
+ if @ignore_lines>0
32
+ @ignore_lines -= 1
33
+ next line
34
+ end
35
+
36
+ row = {}
37
+ line_index = 0
38
+ @schema[:columns].each do |column, column_prop|
39
+ row[column.to_sym] = line[line_index..(line_index+column_prop[:length]-1)]
40
+ row[column.to_sym].strip! if @schema[:strip_columns] == :true && !row[column.to_sym].nil?
41
+ line_index += column_prop[:length]
42
+ end
43
+ row.keys.each {|k| row.delete k if k.to_s =~ /ignore\d+/ }
44
+ puts row.to_yaml
45
+ @migration.migreate_row! row
46
+ end
47
+ file.close
48
+
49
+ end
50
+
51
+ end
52
+
53
+ end
@@ -1,9 +1,9 @@
1
1
 
2
- module ActiveMigration
2
+ module ExternalMigration
3
3
  module Transformer
4
4
  class GroupedFieldFixedSpelling
5
5
 
6
- include ActiveMigration::Transformer
6
+ include ExternalMigration::Transformer
7
7
  include ApplicationHelper
8
8
 
9
9
  attr_accessor :domain_name, :domain_dest
@@ -84,7 +84,7 @@ module ActiveMigration
84
84
 
85
85
  #cached path
86
86
  @cache_path = "db/external_migrate/cache/"
87
- Dir.mkdir path if not Dir.exists? @cache_path
87
+ Dir.mkdir @cache_path if not Dir.exists? @cache_path
88
88
 
89
89
  puts_on_file @cache_path + "#{@domain_name}_dictionary.yml", @domain_dictionary.to_yaml
90
90
  #puts_on_file @cache_path + "#{@domain_name}_raw.csv", @rows.map { |i| i.values.join(",")}.join("\n")
@@ -1,4 +1,6 @@
1
1
 
2
+ require 'spreadsheet'
3
+
2
4
  #main
3
5
  require "external_migration/migration"
4
6
 
@@ -6,5 +8,8 @@ require "external_migration/migration"
6
8
  require "external_migration/dictionary"
7
9
  require "external_migration/spelling_fix"
8
10
 
11
+ #decoders
12
+ require "external_migration/text_fixed"
13
+
9
14
  #schemas
10
15
  require "external_migration/schemas"
@@ -0,0 +1,12 @@
1
+ namespace :db do
2
+ task external_migrate: :environment do
3
+ require "external_migration"
4
+
5
+ Dir[Rails.root.join("db/external_migrate/*schemas.yml")].each do |file|
6
+ @migration_schemas = ExternalMigration::Schemas::SchemasMigration.new file
7
+ if @migration_schemas.migrate!
8
+ Rails.logger.info "Migration success! %s" % file
9
+ end
10
+ end
11
+ end
12
+ end
metadata CHANGED
@@ -1,7 +1,7 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: active_extend
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.0.1
4
+ version: 0.0.2
5
5
  prerelease:
6
6
  platform: ruby
7
7
  authors:
@@ -9,7 +9,7 @@ authors:
9
9
  autorequire:
10
10
  bindir: bin
11
11
  cert_chain: []
12
- date: 2012-12-24 00:00:00.000000000 Z
12
+ date: 2013-04-29 00:00:00.000000000 Z
13
13
  dependencies:
14
14
  - !ruby/object:Gem::Dependency
15
15
  name: rails
@@ -28,14 +28,14 @@ dependencies:
28
28
  - !ruby/object:Gem::Version
29
29
  version: 3.2.9
30
30
  - !ruby/object:Gem::Dependency
31
- name: sqlite3
31
+ name: spreadsheet
32
32
  requirement: !ruby/object:Gem::Requirement
33
33
  none: false
34
34
  requirements:
35
35
  - - ! '>='
36
36
  - !ruby/object:Gem::Version
37
37
  version: '0'
38
- type: :development
38
+ type: :runtime
39
39
  prerelease: false
40
40
  version_requirements: !ruby/object:Gem::Requirement
41
41
  none: false
@@ -51,6 +51,7 @@ extensions: []
51
51
  extra_rdoc_files: []
52
52
  files:
53
53
  - lib/active_extend/active_disablable.rb
54
+ - lib/active_extend/railtie.rb
54
55
  - lib/active_extend/string_helper.rb
55
56
  - lib/active_extend/version.rb
56
57
  - lib/active_extend.rb
@@ -59,9 +60,10 @@ files:
59
60
  - lib/external_migration/migration.rb
60
61
  - lib/external_migration/schemas.rb
61
62
  - lib/external_migration/spelling_fix.rb
63
+ - lib/external_migration/text_fixed.rb
62
64
  - lib/external_migration/transformer/grouped_field_fixed_spelling.rb
63
65
  - lib/external_migration.rb
64
- - lib/tasks/active_extends_tasks.rake
66
+ - lib/tasks/external_migrate.rake
65
67
  - MIT-LICENSE
66
68
  - Rakefile
67
69
  - README.rdoc
@@ -1,4 +0,0 @@
1
- # desc "Explaining what the task does"
2
- # task :active_extend do
3
- # # Task goes here
4
- # end