active_extend 0.0.1 → 0.0.2
Sign up to get free protection for your applications and to get access to all the features.
- data/lib/active_extend/active_disablable.rb +1 -1
- data/lib/active_extend/railtie.rb +10 -0
- data/lib/active_extend/version.rb +1 -1
- data/lib/active_extend.rb +2 -0
- data/lib/external_migration/migration.rb +56 -21
- data/lib/external_migration/schemas.rb +17 -13
- data/lib/external_migration/text_fixed.rb +53 -0
- data/lib/external_migration/transformer/grouped_field_fixed_spelling.rb +3 -3
- data/lib/external_migration.rb +5 -0
- data/lib/tasks/external_migrate.rake +12 -0
- metadata +7 -5
- data/lib/tasks/active_extends_tasks.rake +0 -4
data/lib/active_extend.rb
CHANGED
@@ -1,5 +1,4 @@
|
|
1
1
|
|
2
|
-
require 'spreadsheet'
|
3
2
|
require 'active_support'
|
4
3
|
|
5
4
|
module ExternalMigration
|
@@ -25,7 +24,8 @@ module ExternalMigration
|
|
25
24
|
##
|
26
25
|
# called on start of migration
|
27
26
|
def begin(schema_from, schema_to)
|
28
|
-
|
27
|
+
@schema_from = schema_from
|
28
|
+
@schema_to = schema_to
|
29
29
|
end
|
30
30
|
|
31
31
|
##
|
@@ -66,6 +66,10 @@ module ExternalMigration
|
|
66
66
|
|
67
67
|
end
|
68
68
|
|
69
|
+
module Decoder
|
70
|
+
def migrate!
|
71
|
+
end
|
72
|
+
end
|
69
73
|
|
70
74
|
##
|
71
75
|
# == Class Migration
|
@@ -101,12 +105,18 @@ module ExternalMigration
|
|
101
105
|
|
102
106
|
res = @transformer.begin_transaction(@schema_from, @schema_to) unless @transformer.nil?
|
103
107
|
|
108
|
+
@line = 0
|
109
|
+
|
104
110
|
ActiveRecord::Base.transaction do
|
105
111
|
begin_migration()
|
106
112
|
|
107
113
|
# TODO: Make flexible configurable and more input formats
|
108
114
|
if @schema_from[:format].to_s.to_sym == :XLS
|
109
|
-
|
115
|
+
xls_migrate()
|
116
|
+
elsif @schema_from[:format].to_s.to_sym == :TXT_FIXED
|
117
|
+
decoder = ExternalMigration::TextFixed.new(@schema_from)
|
118
|
+
decoder.migration = self
|
119
|
+
decoder.migrate!
|
110
120
|
end
|
111
121
|
|
112
122
|
end_migration()
|
@@ -117,6 +127,34 @@ module ExternalMigration
|
|
117
127
|
return true
|
118
128
|
end
|
119
129
|
|
130
|
+
def migreate_row!(row_to)
|
131
|
+
@line += 1
|
132
|
+
begin
|
133
|
+
#transform row to @schema_to
|
134
|
+
res = true
|
135
|
+
res = @transformer.transform(row_to) unless @transformer.nil?
|
136
|
+
|
137
|
+
if (res!=:ignore)
|
138
|
+
res = res==true && send_row_to_schema(row_to)
|
139
|
+
raise_migration if (res==false)
|
140
|
+
|
141
|
+
@transformer.after_row_saved(row_to, @last_object) unless @transformer.nil?
|
142
|
+
end
|
143
|
+
rescue Exception => e
|
144
|
+
line = @line.nil? ? 0 : @line
|
145
|
+
column = @column.nil? ? 0 : @column
|
146
|
+
|
147
|
+
obj = @last_object || (e.respond_to?(:record) && (e.record)) || nil
|
148
|
+
|
149
|
+
if !obj.nil?
|
150
|
+
raise ActiveMigartionDataSourceError.new obj.errors.to_yaml.to_s.concat("Failing import excel source format from %s. %d:%d [ignored head]. " % [@schema_from[:url], column, line]).concat(e.message).concat("\n----"+e.backtrace.to_yaml)
|
151
|
+
else
|
152
|
+
raise ActiveMigartionDataSourceError.new ("Failing import excel source format from %s. %d:%d [ignored head]. " % [@schema_from[:url], column, line]).concat(e.message).concat("\n----"+e.backtrace.to_yaml)
|
153
|
+
end
|
154
|
+
end
|
155
|
+
|
156
|
+
end
|
157
|
+
|
120
158
|
|
121
159
|
def xls_migrate
|
122
160
|
begin
|
@@ -124,8 +162,6 @@ module ExternalMigration
|
|
124
162
|
# TODO: make others workbook accessible by configuration
|
125
163
|
sheet = @xls.worksheet 0
|
126
164
|
|
127
|
-
@line = 0
|
128
|
-
|
129
165
|
# ignore head line
|
130
166
|
sheet.each 1 do |row|
|
131
167
|
@column = 0
|
@@ -137,23 +173,19 @@ module ExternalMigration
|
|
137
173
|
@column+=1
|
138
174
|
end
|
139
175
|
|
140
|
-
|
141
|
-
res = true
|
142
|
-
res = @transformer.transform(row_to) unless @transformer.nil?
|
143
|
-
|
144
|
-
if (res!=:ignore)
|
145
|
-
res = res==true && send_row_to_schema(row_to)
|
146
|
-
raise_migration if (res==false)
|
147
|
-
|
148
|
-
@transformer.after_row_saved(row_to, @last_object) unless @transformer.nil?
|
149
|
-
end
|
150
|
-
|
151
|
-
@line+=1
|
176
|
+
self.migrate_row! row_to
|
152
177
|
end
|
153
178
|
rescue Exception => e
|
154
179
|
line = @line.nil? ? 0 : @line
|
155
180
|
column = @column.nil? ? 0 : @column
|
156
|
-
|
181
|
+
|
182
|
+
obj = @last_object || (e.respond_to?(:record) && (e.record)) || nil
|
183
|
+
|
184
|
+
if !obj.nil?
|
185
|
+
raise ActiveMigartionDataSourceError.new obj.errors.to_yaml.to_s.concat("Failing import excel source format from %s. %d:%d [ignored head]. " % [@schema_from[:url], column, line]).concat(e.message).concat("\n----"+e.backtrace.to_yaml)
|
186
|
+
else
|
187
|
+
raise ActiveMigartionDataSourceError.new ("Failing import excel source format from %s. %d:%d [ignored head]. " % [@schema_from[:url], column, line]).concat(e.message).concat("\n----"+e.backtrace.to_yaml)
|
188
|
+
end
|
157
189
|
end
|
158
190
|
end
|
159
191
|
|
@@ -173,16 +205,19 @@ module ExternalMigration
|
|
173
205
|
end
|
174
206
|
|
175
207
|
def send_row_to_schema(row)
|
176
|
-
if @schema_to[:format].to_sym == :ACTIVE_RECORD
|
177
208
|
|
209
|
+
if @schema_to[:format].to_sym == :ACTIVE_RECORD
|
210
|
+
|
178
211
|
# TODO: optimize on initialize migration
|
179
212
|
class_schema_to = eval @schema_to[:url]
|
180
|
-
|
213
|
+
|
181
214
|
@last_object = class_schema_to.new(row)
|
182
215
|
res = @last_object.save
|
183
216
|
|
184
217
|
if (!res)
|
185
|
-
|
218
|
+
msg = "[Schema:%s] Error on send to ACTIVE_RECORD %s. \n%s \nrow: \n%s" % [@name, @schema_to[:url], @last_object.errors.to_yaml, row.to_yaml]
|
219
|
+
Rails.logger.error msg
|
220
|
+
raise AciteMigrationInvalidRecordError.new msg
|
186
221
|
end
|
187
222
|
|
188
223
|
return res
|
@@ -39,13 +39,23 @@ module ExternalMigration
|
|
39
39
|
|
40
40
|
raise ActiveMigartionSchemasError.new("Failing Migrate Schemas: %s" % key) if not result
|
41
41
|
|
42
|
-
msg = "Ending: %s." %
|
42
|
+
msg = "Ending: %s." % @migration_name
|
43
43
|
Rails.logger.info msg
|
44
44
|
puts msg
|
45
45
|
end
|
46
46
|
end
|
47
47
|
end
|
48
48
|
|
49
|
+
def run_migration_job
|
50
|
+
transformer_from_schema()
|
51
|
+
|
52
|
+
case @schema[:type]
|
53
|
+
when :SCHEMA
|
54
|
+
self.migrate_schema
|
55
|
+
when :CUSTOM
|
56
|
+
self.migrate_custom
|
57
|
+
end
|
58
|
+
end
|
49
59
|
|
50
60
|
def eval_class(class_str)
|
51
61
|
begin
|
@@ -57,7 +67,12 @@ module ExternalMigration
|
|
57
67
|
end
|
58
68
|
|
59
69
|
def transformer_class=(class_str)
|
60
|
-
|
70
|
+
|
71
|
+
path = class_str.split("::")
|
72
|
+
|
73
|
+
|
74
|
+
path.map!(&:underscore)
|
75
|
+
file_name = path.join("/") + ".rb"
|
61
76
|
|
62
77
|
class_found = eval_class(class_str)
|
63
78
|
|
@@ -101,17 +116,6 @@ module ExternalMigration
|
|
101
116
|
files
|
102
117
|
end
|
103
118
|
|
104
|
-
def run_migration_job
|
105
|
-
transformer_from_schema
|
106
|
-
|
107
|
-
case @schema[:type]
|
108
|
-
when :SCHEMA
|
109
|
-
self.migrate_schema
|
110
|
-
when :CUSTOM
|
111
|
-
self.migrate_custom
|
112
|
-
end
|
113
|
-
end
|
114
|
-
|
115
119
|
def migrate_schema
|
116
120
|
migration = ExternalMigration::Migration.new
|
117
121
|
migration.name = @migration_name
|
@@ -0,0 +1,53 @@
|
|
1
|
+
|
2
|
+
|
3
|
+
module ExternalMigration
|
4
|
+
|
5
|
+
#
|
6
|
+
# Schema format:
|
7
|
+
#
|
8
|
+
# format: TXT_FIXED
|
9
|
+
# url: text to read
|
10
|
+
# ignore_lines: default 1
|
11
|
+
# encoding: default 'windows-1251:utf-8'
|
12
|
+
class TextFixed
|
13
|
+
include ExternalMigration::Decoder
|
14
|
+
|
15
|
+
attr_accessor :migration, :ignore_lines
|
16
|
+
|
17
|
+
def initialize(schema)
|
18
|
+
self.schema = schema
|
19
|
+
@ignore_lines = schema[:ignore_lines] || 1
|
20
|
+
@enconding = schema[:encoding] || 'windows-1251:utf-8'
|
21
|
+
end
|
22
|
+
|
23
|
+
def schema=(schema)
|
24
|
+
@schema = schema
|
25
|
+
end
|
26
|
+
|
27
|
+
def migrate!
|
28
|
+
puts "opening file #{@schema[:url]}"
|
29
|
+
file = File.open(Rails.root.join(@schema[:url]), "r:#{@enconding}")
|
30
|
+
file.each_line do |line|
|
31
|
+
if @ignore_lines>0
|
32
|
+
@ignore_lines -= 1
|
33
|
+
next line
|
34
|
+
end
|
35
|
+
|
36
|
+
row = {}
|
37
|
+
line_index = 0
|
38
|
+
@schema[:columns].each do |column, column_prop|
|
39
|
+
row[column.to_sym] = line[line_index..(line_index+column_prop[:length]-1)]
|
40
|
+
row[column.to_sym].strip! if @schema[:strip_columns] == :true && !row[column.to_sym].nil?
|
41
|
+
line_index += column_prop[:length]
|
42
|
+
end
|
43
|
+
row.keys.each {|k| row.delete k if k.to_s =~ /ignore\d+/ }
|
44
|
+
puts row.to_yaml
|
45
|
+
@migration.migreate_row! row
|
46
|
+
end
|
47
|
+
file.close
|
48
|
+
|
49
|
+
end
|
50
|
+
|
51
|
+
end
|
52
|
+
|
53
|
+
end
|
@@ -1,9 +1,9 @@
|
|
1
1
|
|
2
|
-
module
|
2
|
+
module ExternalMigration
|
3
3
|
module Transformer
|
4
4
|
class GroupedFieldFixedSpelling
|
5
5
|
|
6
|
-
include
|
6
|
+
include ExternalMigration::Transformer
|
7
7
|
include ApplicationHelper
|
8
8
|
|
9
9
|
attr_accessor :domain_name, :domain_dest
|
@@ -84,7 +84,7 @@ module ActiveMigration
|
|
84
84
|
|
85
85
|
#cached path
|
86
86
|
@cache_path = "db/external_migrate/cache/"
|
87
|
-
Dir.mkdir
|
87
|
+
Dir.mkdir @cache_path if not Dir.exists? @cache_path
|
88
88
|
|
89
89
|
puts_on_file @cache_path + "#{@domain_name}_dictionary.yml", @domain_dictionary.to_yaml
|
90
90
|
#puts_on_file @cache_path + "#{@domain_name}_raw.csv", @rows.map { |i| i.values.join(",")}.join("\n")
|
data/lib/external_migration.rb
CHANGED
@@ -1,4 +1,6 @@
|
|
1
1
|
|
2
|
+
require 'spreadsheet'
|
3
|
+
|
2
4
|
#main
|
3
5
|
require "external_migration/migration"
|
4
6
|
|
@@ -6,5 +8,8 @@ require "external_migration/migration"
|
|
6
8
|
require "external_migration/dictionary"
|
7
9
|
require "external_migration/spelling_fix"
|
8
10
|
|
11
|
+
#decoders
|
12
|
+
require "external_migration/text_fixed"
|
13
|
+
|
9
14
|
#schemas
|
10
15
|
require "external_migration/schemas"
|
@@ -0,0 +1,12 @@
|
|
1
|
+
namespace :db do
|
2
|
+
task external_migrate: :environment do
|
3
|
+
require "external_migration"
|
4
|
+
|
5
|
+
Dir[Rails.root.join("db/external_migrate/*schemas.yml")].each do |file|
|
6
|
+
@migration_schemas = ExternalMigration::Schemas::SchemasMigration.new file
|
7
|
+
if @migration_schemas.migrate!
|
8
|
+
Rails.logger.info "Migration success! %s" % file
|
9
|
+
end
|
10
|
+
end
|
11
|
+
end
|
12
|
+
end
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: active_extend
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.0.
|
4
|
+
version: 0.0.2
|
5
5
|
prerelease:
|
6
6
|
platform: ruby
|
7
7
|
authors:
|
@@ -9,7 +9,7 @@ authors:
|
|
9
9
|
autorequire:
|
10
10
|
bindir: bin
|
11
11
|
cert_chain: []
|
12
|
-
date:
|
12
|
+
date: 2013-04-29 00:00:00.000000000 Z
|
13
13
|
dependencies:
|
14
14
|
- !ruby/object:Gem::Dependency
|
15
15
|
name: rails
|
@@ -28,14 +28,14 @@ dependencies:
|
|
28
28
|
- !ruby/object:Gem::Version
|
29
29
|
version: 3.2.9
|
30
30
|
- !ruby/object:Gem::Dependency
|
31
|
-
name:
|
31
|
+
name: spreadsheet
|
32
32
|
requirement: !ruby/object:Gem::Requirement
|
33
33
|
none: false
|
34
34
|
requirements:
|
35
35
|
- - ! '>='
|
36
36
|
- !ruby/object:Gem::Version
|
37
37
|
version: '0'
|
38
|
-
type: :
|
38
|
+
type: :runtime
|
39
39
|
prerelease: false
|
40
40
|
version_requirements: !ruby/object:Gem::Requirement
|
41
41
|
none: false
|
@@ -51,6 +51,7 @@ extensions: []
|
|
51
51
|
extra_rdoc_files: []
|
52
52
|
files:
|
53
53
|
- lib/active_extend/active_disablable.rb
|
54
|
+
- lib/active_extend/railtie.rb
|
54
55
|
- lib/active_extend/string_helper.rb
|
55
56
|
- lib/active_extend/version.rb
|
56
57
|
- lib/active_extend.rb
|
@@ -59,9 +60,10 @@ files:
|
|
59
60
|
- lib/external_migration/migration.rb
|
60
61
|
- lib/external_migration/schemas.rb
|
61
62
|
- lib/external_migration/spelling_fix.rb
|
63
|
+
- lib/external_migration/text_fixed.rb
|
62
64
|
- lib/external_migration/transformer/grouped_field_fixed_spelling.rb
|
63
65
|
- lib/external_migration.rb
|
64
|
-
- lib/tasks/
|
66
|
+
- lib/tasks/external_migrate.rake
|
65
67
|
- MIT-LICENSE
|
66
68
|
- Rakefile
|
67
69
|
- README.rdoc
|