mssql_mysql_migrator 0.1.4 → 0.1.5

Sign up to get free protection for your applications and to get access to all the features.
Files changed (6) hide show
  1. checksums.yaml +4 -4
  2. data/Gemfile.lock +7 -7
  3. data/exe/mssql-mysql-migrator +35 -16
  4. data/lib/mssql_mysql_migrator/version.rb +1 -1
  5. data/mssql_mysql_migrator.gemspec +9 -12
  6. metadata +34 -34
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: c0bcb64c9fef0d76d644a351b29755f31abf68d3
4
- data.tar.gz: da2600300941f1648553aaf439bf8328bac9b1dc
3
+ metadata.gz: fedff8aeb75df313751c0ebdac9300142dff5b7d
4
+ data.tar.gz: b7d6ce173e45aa973875d0a5567c40285fb87e33
5
5
  SHA512:
6
- metadata.gz: 5f4a25be62eea077cdaae6c61acb749d61656392e086bb6ae2c4e83c2778cf65d3e8c12e73c0be951992a784e2498a53cd52ffa89c0b400360cf86072338725f
7
- data.tar.gz: e1da9c0bf3935753e91d69aee52c5f2f47679719b536c4809bf9e80bb88093b9002c416888df12fdadf0ed8dc21e5d77170a344310b2b8d0df3bed23fe764b5a
6
+ metadata.gz: 3620c9a610bd97ac9402863c2b68564daa62e553c590b5c54f24bf331bf7459e9322f5acc5b781bf25314f8b68f491e68f8202608a92777ea70d7aa906366ce4
7
+ data.tar.gz: e01d18e9e442d8dbef81106f205fc444e66bacb8af34ad263965616b30d00e02a26ec282fda21a2ae137534bc3db2ec8225579e002c8ca4acd72d0dca7ec8cc4
data/Gemfile.lock CHANGED
@@ -1,13 +1,12 @@
1
1
  PATH
2
2
  remote: .
3
3
  specs:
4
- mssql_mysql_migrator (0.1.4)
4
+ mssql_mysql_migrator (0.1.5)
5
5
  ac-logger (~> 0.1.1)
6
6
  applix (~> 0.4.14)
7
7
  bundler (~> 1.10)
8
- mysql
9
- rake (~> 10.0)
10
- sequel
8
+ mysql (~> 2.9)
9
+ sequel (~> 4.22)
11
10
  tiny_tds (~> 0.6.3.rc1)
12
11
 
13
12
  GEM
@@ -43,9 +42,10 @@ PLATFORMS
43
42
  ruby
44
43
 
45
44
  DEPENDENCIES
46
- byebug
45
+ byebug (~> 5)
47
46
  mssql_mysql_migrator!
48
- rspec
47
+ rake (~> 10.0)
48
+ rspec (~> 3)
49
49
 
50
50
  BUNDLED WITH
51
- 1.10.3
51
+ 1.10.5
data/exe/mssql-mysql-migrator CHANGED
@@ -2,10 +2,9 @@
2
2
 
3
3
  require 'rubygems'
4
4
  require 'bundler'
5
- Bundler.setup(:default) #require 'bundler/setup'
6
- require 'ac/logger'
7
- require 'applix'
5
+ Bundler.setup
8
6
 
7
+ require 'applix'
9
8
  require 'mssql_mysql_migrator'
10
9
 
11
10
  $log = AC::Logger[]
@@ -53,20 +52,18 @@ end
53
52
 
54
53
  class MySQL < SequelDB
55
54
  def initialize params
56
- debugger rescue nil
57
55
  super(params.merge(adapter: 'mysql'))
58
56
  end
59
57
 
60
58
  def import_table(table_name, csv_path)
59
+ ds = db.run("DELETE FROM #{table_name}")
61
60
  sql = <<-SQL
62
- DELETE FROM #{table_name};
63
61
  LOAD DATA INFILE '#{csv_path}' INTO TABLE #{table_name}
64
62
  FIELDS TERMINATED BY ',' OPTIONALLY ENCLOSED BY '\"'
65
63
  IGNORE 1 LINES;
66
64
  SHOW WARNINGS
67
65
  SQL
68
66
  $log.debug "running mysql import: #{sql}"
69
- debugger rescue nil
70
67
  ds = db.run(sql)
71
68
  end
72
69
  end
@@ -108,9 +105,19 @@ class Main
108
105
  #
109
106
  # ... import-table test ad_locations $(pwd)/tmp/ad_locations.csv
110
107
  #
108
+ # options:
109
+ #
110
+ # --loaddir=<path> take path basename and prefix it with loaddir (this
111
+ # actually make sense in case of using transfer table)
112
+ #
111
113
  def import_table(db_name, table_name, csv_path)
112
114
  $log.debug "open destination datbabase(#{db_name})"
113
115
  dest_db = open_destination_db(db_name)
116
+
117
+ filename = File.basename(csv_path)
118
+ dirname = (@config[:loaddir] || File.dirname(csv_path))
119
+ csv_path = File.join(dirname, filename)
120
+
114
121
  dest_db.import_table(table_name, csv_path)
115
122
  end
116
123
 
@@ -135,8 +142,7 @@ class Main
135
142
  dump_table_as_csv(fd, table)
136
143
  fd.close
137
144
 
138
- dest_db = open_destination_db(name_dst)
139
- dest_db.import_table(table_name, fd.path)
145
+ import_table(name_dst, table_name, fd.path)
140
146
 
141
147
  []
142
148
  end
@@ -155,6 +161,10 @@ class Main
155
161
  #
156
162
  # when no table list is given all tables from src db are transfered
157
163
  #
164
+ # options:
165
+ #
166
+ # --loaddir=<path> take path basename and prefix it with loaddir
167
+ #
158
168
  def transfer(name_src, name_dst, *args)
159
169
  $log.debug "open source datbabase(#{name_src})"
160
170
  source_db= open_source_db(name_src)
@@ -208,17 +218,27 @@ class Main
208
218
  $log.debug "dumping #{table.count} rows as csv"
209
219
  table.each do |row|
210
220
  begin
211
- #debugger if row[:id] == 7047
212
- #r = c_names.map {|c| row[c]}
213
- #csv << prep_values(r)
214
221
  csv << prep_values(c_names.map {|c| row[c]})
215
222
  rescue => e
216
- debugger rescue nil
217
- $log.warn "bad row? #{row} -- #{e}"
223
+ $log.warn "bad row? #{row} -- #{e.class}::#{e}"
224
+ $log.debug "(callstack):\n\t#{e.backtrace.join "\n\t"}"
218
225
  end
219
226
  end
220
227
  end
221
228
 
229
+ # XXX: THIS IS A HORROR! DIRTY FIX!!! only justified by the fact that it is
230
+ # needed to repair a multiple broken encoding on strings in the current live
231
+ # database.
232
+ #
233
+ def reverse_totally_broken_string_data_encoding(val)
234
+ c = val.encode('CP1252').force_encoding('utf-8')
235
+ c.count('.')
236
+ c
237
+ rescue ArgumentError => e
238
+ $log.warn "encoding problem: >#{val}<>#{val.bytes}< -- #{e.class}::#{e}"
239
+ val.encode('utf-8')
240
+ end
241
+
222
242
  def prep_values(values)
223
243
  # In Csv I replaced all the "" with "\N" before I run the script, this
224
244
  # creates a null field in db
@@ -230,10 +250,9 @@ class Main
230
250
  when Time
231
251
  val.strftime("%Y-%m-%d %H:%M:%S")
232
252
  when String
233
- val.encode('CP1252').force_encoding('utf-8')
234
- #val.force_encoding('utf-8')
253
+ reverse_totally_broken_string_data_encoding(val)
235
254
  when true, false
236
- val ? 0 : 1
255
+ val ? 1 : 0
237
256
  else
238
257
  val
239
258
  end
data/lib/mssql_mysql_migrator/version.rb CHANGED
@@ -1,3 +1,3 @@
1
1
  module MssqlMysqlMigrator
2
- VERSION = "0.1.4"
2
+ VERSION = '0.1.5'
3
3
  end
data/mssql_mysql_migrator.gemspec CHANGED
@@ -27,17 +27,14 @@ Gem::Specification.new do |spec|
27
27
  spec.executables = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
28
28
  spec.require_paths = ["lib"]
29
29
 
30
- spec.add_dependency 'applix', '~>0.4.14'
31
- spec.add_dependency 'ac-logger', '~>0.1.1'
30
+ spec.add_runtime_dependency "bundler", "~> 1.10"
31
+ spec.add_runtime_dependency 'applix', '~>0.4.14'
32
+ spec.add_runtime_dependency 'ac-logger', '~>0.1.1'
33
+ spec.add_runtime_dependency 'mysql', '~>2.9'
34
+ spec.add_runtime_dependency 'sequel', '~>4.22'
35
+ spec.add_runtime_dependency 'tiny_tds', '~> 0.6.3.rc1'
32
36
 
33
- spec.add_dependency 'mysql'
34
- spec.add_dependency 'sequel'
35
- spec.add_dependency 'tiny_tds', '~> 0.6.3.rc1'
36
- spec.add_dependency "bundler", "~> 1.10"
37
- spec.add_dependency "rake", "~> 10.0"
38
- ## MSSQL for production server connection
39
- #spec.add_dependency 'activerecord-sqlserver-adapter'
40
-
41
- spec.add_development_dependency "rspec"
42
- spec.add_development_dependency "byebug"
37
+ spec.add_development_dependency "rake", "~> 10.0"
38
+ spec.add_development_dependency "rspec", "~> 3"
39
+ spec.add_development_dependency "byebug", "~> 5"
43
40
  end
metadata CHANGED
@@ -1,99 +1,99 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: mssql_mysql_migrator
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.1.4
4
+ version: 0.1.5
5
5
  platform: ruby
6
6
  authors:
7
7
  - dirk lüsebrink
8
8
  autorequire:
9
9
  bindir: exe
10
10
  cert_chain: []
11
- date: 2015-06-23 00:00:00.000000000 Z
11
+ date: 2015-06-29 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
- name: applix
14
+ name: bundler
15
15
  requirement: !ruby/object:Gem::Requirement
16
16
  requirements:
17
17
  - - "~>"
18
18
  - !ruby/object:Gem::Version
19
- version: 0.4.14
19
+ version: '1.10'
20
20
  type: :runtime
21
21
  prerelease: false
22
22
  version_requirements: !ruby/object:Gem::Requirement
23
23
  requirements:
24
24
  - - "~>"
25
25
  - !ruby/object:Gem::Version
26
- version: 0.4.14
26
+ version: '1.10'
27
27
  - !ruby/object:Gem::Dependency
28
- name: ac-logger
28
+ name: applix
29
29
  requirement: !ruby/object:Gem::Requirement
30
30
  requirements:
31
31
  - - "~>"
32
32
  - !ruby/object:Gem::Version
33
- version: 0.1.1
33
+ version: 0.4.14
34
34
  type: :runtime
35
35
  prerelease: false
36
36
  version_requirements: !ruby/object:Gem::Requirement
37
37
  requirements:
38
38
  - - "~>"
39
39
  - !ruby/object:Gem::Version
40
- version: 0.1.1
40
+ version: 0.4.14
41
41
  - !ruby/object:Gem::Dependency
42
- name: mysql
42
+ name: ac-logger
43
43
  requirement: !ruby/object:Gem::Requirement
44
44
  requirements:
45
- - - ">="
45
+ - - "~>"
46
46
  - !ruby/object:Gem::Version
47
- version: '0'
47
+ version: 0.1.1
48
48
  type: :runtime
49
49
  prerelease: false
50
50
  version_requirements: !ruby/object:Gem::Requirement
51
51
  requirements:
52
- - - ">="
52
+ - - "~>"
53
53
  - !ruby/object:Gem::Version
54
- version: '0'
54
+ version: 0.1.1
55
55
  - !ruby/object:Gem::Dependency
56
- name: sequel
56
+ name: mysql
57
57
  requirement: !ruby/object:Gem::Requirement
58
58
  requirements:
59
- - - ">="
59
+ - - "~>"
60
60
  - !ruby/object:Gem::Version
61
- version: '0'
61
+ version: '2.9'
62
62
  type: :runtime
63
63
  prerelease: false
64
64
  version_requirements: !ruby/object:Gem::Requirement
65
65
  requirements:
66
- - - ">="
66
+ - - "~>"
67
67
  - !ruby/object:Gem::Version
68
- version: '0'
68
+ version: '2.9'
69
69
  - !ruby/object:Gem::Dependency
70
- name: tiny_tds
70
+ name: sequel
71
71
  requirement: !ruby/object:Gem::Requirement
72
72
  requirements:
73
73
  - - "~>"
74
74
  - !ruby/object:Gem::Version
75
- version: 0.6.3.rc1
75
+ version: '4.22'
76
76
  type: :runtime
77
77
  prerelease: false
78
78
  version_requirements: !ruby/object:Gem::Requirement
79
79
  requirements:
80
80
  - - "~>"
81
81
  - !ruby/object:Gem::Version
82
- version: 0.6.3.rc1
82
+ version: '4.22'
83
83
  - !ruby/object:Gem::Dependency
84
- name: bundler
84
+ name: tiny_tds
85
85
  requirement: !ruby/object:Gem::Requirement
86
86
  requirements:
87
87
  - - "~>"
88
88
  - !ruby/object:Gem::Version
89
- version: '1.10'
89
+ version: 0.6.3.rc1
90
90
  type: :runtime
91
91
  prerelease: false
92
92
  version_requirements: !ruby/object:Gem::Requirement
93
93
  requirements:
94
94
  - - "~>"
95
95
  - !ruby/object:Gem::Version
96
- version: '1.10'
96
+ version: 0.6.3.rc1
97
97
  - !ruby/object:Gem::Dependency
98
98
  name: rake
99
99
  requirement: !ruby/object:Gem::Requirement
@@ -101,7 +101,7 @@ dependencies:
101
101
  - - "~>"
102
102
  - !ruby/object:Gem::Version
103
103
  version: '10.0'
104
- type: :runtime
104
+ type: :development
105
105
  prerelease: false
106
106
  version_requirements: !ruby/object:Gem::Requirement
107
107
  requirements:
@@ -112,30 +112,30 @@ dependencies:
112
112
  name: rspec
113
113
  requirement: !ruby/object:Gem::Requirement
114
114
  requirements:
115
- - - ">="
115
+ - - "~>"
116
116
  - !ruby/object:Gem::Version
117
- version: '0'
117
+ version: '3'
118
118
  type: :development
119
119
  prerelease: false
120
120
  version_requirements: !ruby/object:Gem::Requirement
121
121
  requirements:
122
- - - ">="
122
+ - - "~>"
123
123
  - !ruby/object:Gem::Version
124
- version: '0'
124
+ version: '3'
125
125
  - !ruby/object:Gem::Dependency
126
126
  name: byebug
127
127
  requirement: !ruby/object:Gem::Requirement
128
128
  requirements:
129
- - - ">="
129
+ - - "~>"
130
130
  - !ruby/object:Gem::Version
131
- version: '0'
131
+ version: '5'
132
132
  type: :development
133
133
  prerelease: false
134
134
  version_requirements: !ruby/object:Gem::Requirement
135
135
  requirements:
136
- - - ">="
136
+ - - "~>"
137
137
  - !ruby/object:Gem::Version
138
- version: '0'
138
+ version: '5'
139
139
  description: Sequel is used for reading and CSV for bulk uploading.
140
140
  email:
141
141
  - dirk.luesebrink@artcom.de
@@ -180,7 +180,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
180
180
  version: '0'
181
181
  requirements: []
182
182
  rubyforge_project:
183
- rubygems_version: 2.4.6
183
+ rubygems_version: 2.4.8
184
184
  signing_key:
185
185
  specification_version: 4
186
186
  summary: dumping from MS SQL Server and loading to MySQL.