mssql_mysql_migrator 0.1.4 → 0.1.5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: c0bcb64c9fef0d76d644a351b29755f31abf68d3
-  data.tar.gz: da2600300941f1648553aaf439bf8328bac9b1dc
+  metadata.gz: fedff8aeb75df313751c0ebdac9300142dff5b7d
+  data.tar.gz: b7d6ce173e45aa973875d0a5567c40285fb87e33
 SHA512:
-  metadata.gz: 5f4a25be62eea077cdaae6c61acb749d61656392e086bb6ae2c4e83c2778cf65d3e8c12e73c0be951992a784e2498a53cd52ffa89c0b400360cf86072338725f
-  data.tar.gz: e1da9c0bf3935753e91d69aee52c5f2f47679719b536c4809bf9e80bb88093b9002c416888df12fdadf0ed8dc21e5d77170a344310b2b8d0df3bed23fe764b5a
+  metadata.gz: 3620c9a610bd97ac9402863c2b68564daa62e553c590b5c54f24bf331bf7459e9322f5acc5b781bf25314f8b68f491e68f8202608a92777ea70d7aa906366ce4
+  data.tar.gz: e01d18e9e442d8dbef81106f205fc444e66bacb8af34ad263965616b30d00e02a26ec282fda21a2ae137534bc3db2ec8225579e002c8ca4acd72d0dca7ec8cc4

data/Gemfile.lock

@@ -1,13 +1,12 @@
 PATH
   remote: .
   specs:
-    mssql_mysql_migrator (0.1.4)
+    mssql_mysql_migrator (0.1.5)
       ac-logger (~> 0.1.1)
       applix (~> 0.4.14)
       bundler (~> 1.10)
-      mysql
-      rake (~> 10.0)
-      sequel
+      mysql (~> 2.9)
+      sequel (~> 4.22)
       tiny_tds (~> 0.6.3.rc1)
 
 GEM
@@ -43,9 +42,10 @@ PLATFORMS
   ruby
 
 DEPENDENCIES
-  byebug
+  byebug (~> 5)
   mssql_mysql_migrator!
-  rspec
+  rake (~> 10.0)
+  rspec (~> 3)
 
 BUNDLED WITH
-   1.10.3
+   1.10.5

data/exe/mssql-mysql-migrator

@@ -2,10 +2,9 @@
 
 require 'rubygems'
 require 'bundler'
-Bundler.setup(:default) #require 'bundler/setup'
-require 'ac/logger'
-require 'applix'
+Bundler.setup
 
+require 'applix'
 require 'mssql_mysql_migrator'
 
 $log = AC::Logger[]
@@ -53,20 +52,18 @@ end
 
 class MySQL < SequelDB
   def initialize params 
-    debugger rescue nil
     super(params.merge(adapter: 'mysql'))
   end
 
   def import_table(table_name, csv_path)
+    ds = db.run("DELETE FROM #{table_name}")
     sql = <<-SQL
-DELETE FROM #{table_name};
 LOAD DATA INFILE '#{csv_path}' INTO TABLE #{table_name}
   FIELDS TERMINATED BY ',' OPTIONALLY ENCLOSED BY '\"'
   IGNORE 1 LINES;
   SHOW WARNINGS
     SQL
     $log.debug "running mysql import: #{sql}"
-    debugger rescue nil
     ds = db.run(sql)
   end
 end
@@ -108,9 +105,19 @@ class Main
   #
   # ... import-table test  ad_locations $(pwd)/tmp/ad_locations.csv 
   #
+  # options: 
+  #
+  #   --loaddir=<path>    take path basename and prefix it with loaddir (this
+  #                       actually make sense in case of using transfer table)
+  #
   def import_table(db_name, table_name, csv_path)
     $log.debug "open destination datbabase(#{db_name})"
     dest_db = open_destination_db(db_name)
+
+    filename = File.basename(csv_path)
+    dirname = (@config[:loaddir] || File.dirname(csv_path))
+    csv_path = File.join(dirname, filename)
+
     dest_db.import_table(table_name, csv_path)
   end
 
@@ -135,8 +142,7 @@ class Main
     dump_table_as_csv(fd, table)
     fd.close
 
-    dest_db = open_destination_db(name_dst)
-    dest_db.import_table(table_name, fd.path)
+    import_table(name_dst, table_name, fd.path)
 
     []
   end
@@ -155,6 +161,10 @@ class Main
   #
   # when no table list is given all tables from src db are transfered
   #
+  # options:
+  #
+  #   --loaddir=<path>    take path basename and prefix it with loaddir
+  #
   def transfer(name_src, name_dst, *args)
     $log.debug "open source datbabase(#{name_src})"
     source_db= open_source_db(name_src)
@@ -208,17 +218,27 @@ class Main
     $log.debug "dumping #{table.count} rows as csv"
     table.each do |row|
       begin
-        #debugger if row[:id] == 7047
-        #r = c_names.map {|c| row[c]}
-        #csv << prep_values(r)
         csv << prep_values(c_names.map {|c| row[c]})
       rescue => e
-        debugger rescue nil
-        $log.warn "bad row? #{row} -- #{e}"
+        $log.warn "bad row? #{row} -- #{e.class}::#{e}"
+        $log.debug "(callstack):\n\t#{e.backtrace.join "\n\t"}"
       end
     end
   end
 
+  # XXX: THIS IS A HORROR! DIRTY FIX!!! only justified by the fact that it is
+  # needed to repair a multiple broken encoding on strings in the current live
+  # database.
+  # 
+  def reverse_totally_broken_string_data_encoding(val)
+    c = val.encode('CP1252').force_encoding('utf-8')
+    c.count('.')
+    c
+  rescue ArgumentError => e
+    $log.warn "encoding problem: >#{val}<>#{val.bytes}< -- #{e.class}::#{e}"
+    val.encode('utf-8')
+  end
+
   def prep_values(values)
     # In Csv I replaced all the "" with "\N" before I run the script, this
     # creates a null field in db
@@ -230,10 +250,9 @@ class Main
           when Time
             val.strftime("%Y-%m-%d %H:%M:%S")
           when String
-           val.encode('CP1252').force_encoding('utf-8')
-           #val.force_encoding('utf-8')
+            reverse_totally_broken_string_data_encoding(val)
           when true, false
-            val ? 0 : 1
+            val ? 1 : 0
           else
             val
           end

data/lib/mssql_mysql_migrator/version.rb

@@ -1,3 +1,3 @@
 module MssqlMysqlMigrator
-  VERSION = "0.1.4"
+  VERSION = '0.1.5'
 end

data/mssql_mysql_migrator.gemspec

@@ -27,17 +27,14 @@ Gem::Specification.new do |spec|
   spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
   spec.require_paths = ["lib"]
 
-  spec.add_dependency 'applix', '~>0.4.14'
-  spec.add_dependency 'ac-logger', '~>0.1.1'
+  spec.add_runtime_dependency "bundler", "~> 1.10"
+  spec.add_runtime_dependency 'applix', '~>0.4.14'
+  spec.add_runtime_dependency 'ac-logger', '~>0.1.1'
+  spec.add_runtime_dependency 'mysql', '~>2.9'
+  spec.add_runtime_dependency 'sequel', '~>4.22'
+  spec.add_runtime_dependency 'tiny_tds', '~> 0.6.3.rc1'
 
-  spec.add_dependency 'mysql'
-  spec.add_dependency 'sequel'
-  spec.add_dependency 'tiny_tds', '~> 0.6.3.rc1'
-  spec.add_dependency "bundler", "~> 1.10"
-  spec.add_dependency "rake", "~> 10.0"
-  ## MSSQL for production server connection
-  #spec.add_dependency 'activerecord-sqlserver-adapter'
-
-  spec.add_development_dependency "rspec"
-  spec.add_development_dependency "byebug"
+  spec.add_development_dependency "rake", "~> 10.0"
+  spec.add_development_dependency "rspec", "~> 3"
+  spec.add_development_dependency "byebug", "~> 5"
 end

metadata

@@ -1,99 +1,99 @@
 --- !ruby/object:Gem::Specification
 name: mssql_mysql_migrator
 version: !ruby/object:Gem::Version
-  version: 0.1.4
+  version: 0.1.5
 platform: ruby
 authors:
 - dirk lüsebrink
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2015-06-23 00:00:00.000000000 Z
+date: 2015-06-29 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
-  name: applix
+  name: bundler
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.4.14
+        version: '1.10'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.4.14
+        version: '1.10'
 - !ruby/object:Gem::Dependency
-  name: ac-logger
+  name: applix
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.1.1
+        version: 0.4.14
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.1.1
+        version: 0.4.14
 - !ruby/object:Gem::Dependency
-  name: mysql
+  name: ac-logger
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 0.1.1
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 0.1.1
 - !ruby/object:Gem::Dependency
-  name: sequel
+  name: mysql
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '2.9'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '2.9'
 - !ruby/object:Gem::Dependency
-  name: tiny_tds
+  name: sequel
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.6.3.rc1
+        version: '4.22'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.6.3.rc1
+        version: '4.22'
 - !ruby/object:Gem::Dependency
-  name: bundler
+  name: tiny_tds
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.10'
+        version: 0.6.3.rc1
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.10'
+        version: 0.6.3.rc1
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
@@ -101,7 +101,7 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '10.0'
-  type: :runtime
+  type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
@@ -112,30 +112,30 @@ dependencies:
   name: rspec
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '3'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '3'
 - !ruby/object:Gem::Dependency
   name: byebug
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '5'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '5'
 description: Sequel is used for reading and CSV for bulk uploading.
 email:
 - dirk.luesebrink@artcom.de
@@ -180,7 +180,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.4.6
+rubygems_version: 2.4.8
 signing_key: 
 specification_version: 4
 summary: dumping from MS SQL Server and loading to MySQL.