sq-dbsync 1.0.3 → 1.0.4

data/README.md

@@ -15,6 +15,12 @@ existing solutions were able to do this adequately.
 At some point you will need to bite the bullet and implement a real ETL system,
 but `sq-dbsync` can tide you over until you get there.
 
+dbsync is MySQL `utf8mb4` clean: it will correctly handle four-byte
+UTF8 characters like emojis. Under JRuby, you'll need to have the
+server character set configured to `utf8mb4`. The specs include tests
+for this; they'll fail if you run them under JRuby against MySQL with
+different server character set.
+
 Usage
 -----
 

data/lib/sq/dbsync/database/common.rb

@@ -21,7 +21,15 @@ module Sq::Dbsync::Database
         query = query.filter("#{plan.timestamp} > ?", last_row_at - overlap)
       end
 
-      extract_sql_to_file(query.sql, file_name)
+      sql = query.sql
+
+      sql = customize_sql(sql, db.schema(table_name))
+
+      extract_sql_to_file(sql, file_name)
+    end
+
+    def customize_sql(sql, schema)
+      sql
     end
 
     def hash_schema(table_name)

data/lib/sq/dbsync/database/postgres.rb

@@ -26,6 +26,7 @@ module Sq::Dbsync::Database
       ensure_connection
 
       result = schema(table_name).each do |col, metadata|
+        metadata[:source_db_type] ||= metadata[:db_type]
         metadata[:db_type] = psql_to_mysql_conversion(metadata[:db_type])
       end
 
@@ -59,6 +60,21 @@ module Sq::Dbsync::Database
       }.fetch(db_type, db_type)
     end
 
+
+    # 'with time zone' pg fields have to be selected as
+    # "column-name"::timestamp or they end up with a +XX tz offset on
+    # them, which isn't the correct format for timestamp fields (which
+    # is what they get turned into for portability.)
+
+    def customize_sql(sql, schema)
+      schema.each do |name, metadata|
+        if metadata[:source_db_type].end_with? "with time zone"
+          sql.sub! %r{"#{name}"}, '\0::timestamp'
+        end
+      end
+      sql
+    end
+
     def extract_sql_to_file(sql, file_name)
       sql = "COPY (#{sql}) TO STDOUT"
       file = sql_to_file(sql)

data/lib/sq/dbsync/version.rb

@@ -1,5 +1,5 @@
 module Sq
   module Dbsync
-    VERSION = '1.0.3'
+    VERSION = '1.0.4'
   end
 end

data/spec/integration/batch_load_action_spec.rb

@@ -203,21 +203,6 @@ describe SQD::BatchLoadAction do
     let(:source) { test_source(:source) }
 
     it_should_behave_like 'a batch load'
-
-    it 'loads records with bodgy timestamps' do
-      create_source_table_with(
-        id:         1,
-        col1:       'hello',
-        pii:        'don alias',
-        created_at: '0000-00-00 00:00:00',
-        updated_at: '0000-00-00 00:00:00'
-      )
-      registry.ensure_storage_exists
-
-      action.call
-
-      target[:test_table].count.should == 1
-    end
   end
 
   describe 'with PG source' do

metadata

@@ -1,105 +1,105 @@
---- !ruby/object:Gem::Specification 
+--- !ruby/object:Gem::Specification
 name: sq-dbsync
-version: !ruby/object:Gem::Version 
-  hash: 17
+version: !ruby/object:Gem::Version
+  version: 1.0.4
   prerelease: 
-  segments: 
-  - 1
-  - 0
-  - 3
-  version: 1.0.3
 platform: ruby
-authors: 
+authors:
 - Xavier Shay
 - Damon McCormick
 autorequire: 
 bindir: bin
 cert_chain: []
-
-date: 2013-04-16 00:00:00 Z
-dependencies: 
-- !ruby/object:Gem::Dependency 
+date: 2013-04-17 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
   name: rspec
-  prerelease: false
-  requirement: &id001 !ruby/object:Gem::Requirement 
+  requirement: !ruby/object:Gem::Requirement
     none: false
-    requirements: 
+    requirements:
     - - ~>
-      - !ruby/object:Gem::Version 
-        hash: 3
-        segments: 
-        - 2
-        - 0
-        version: "2.0"
+      - !ruby/object:Gem::Version
+        version: '2.0'
   type: :development
-  version_requirements: *id001
-- !ruby/object:Gem::Dependency 
-  name: rake
   prerelease: false
-  requirement: &id002 !ruby/object:Gem::Requirement 
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '2.0'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
     none: false
-    requirements: 
-    - - ">="
-      - !ruby/object:Gem::Version 
-        hash: 3
-        segments: 
-        - 0
-        version: "0"
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
   type: :development
-  version_requirements: *id002
-- !ruby/object:Gem::Dependency 
-  name: simplecov
   prerelease: false
-  requirement: &id003 !ruby/object:Gem::Requirement 
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: simplecov
+  requirement: !ruby/object:Gem::Requirement
     none: false
-    requirements: 
-    - - ">="
-      - !ruby/object:Gem::Version 
-        hash: 3
-        segments: 
-        - 0
-        version: "0"
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
   type: :development
-  version_requirements: *id003
-- !ruby/object:Gem::Dependency 
-  name: cane
   prerelease: false
-  requirement: &id004 !ruby/object:Gem::Requirement 
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: cane
+  requirement: !ruby/object:Gem::Requirement
     none: false
-    requirements: 
-    - - ">="
-      - !ruby/object:Gem::Version 
-        hash: 3
-        segments: 
-        - 0
-        version: "0"
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
   type: :development
-  version_requirements: *id004
-- !ruby/object:Gem::Dependency 
-  name: sequel
   prerelease: false
-  requirement: &id005 !ruby/object:Gem::Requirement 
+  version_requirements: !ruby/object:Gem::Requirement
     none: false
-    requirements: 
-    - - ">="
-      - !ruby/object:Gem::Version 
-        hash: 3
-        segments: 
-        - 0
-        version: "0"
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: sequel
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
   type: :runtime
-  version_requirements: *id005
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
 description: Column based, timestamp replication of MySQL and Postgres databases.
-email: 
+email:
 - xavier@squareup.com
 - damon@squaerup.com
 executables: []
-
 extensions: []
-
 extra_rdoc_files: []
-
-files: 
+files:
 - spec/acceptance/loading_spec.rb
 - spec/acceptance_helper.rb
 - spec/database_helper.rb
@@ -145,38 +145,30 @@ files:
 - sq-dbsync.gemspec
 homepage: http://github.com/square/sq-dbsync
 licenses: []
-
 post_install_message: 
 rdoc_options: []
-
-require_paths: 
+require_paths:
 - lib
-required_ruby_version: !ruby/object:Gem::Requirement 
+required_ruby_version: !ruby/object:Gem::Requirement
   none: false
-  requirements: 
-  - - ">="
-    - !ruby/object:Gem::Version 
-      hash: 3
-      segments: 
-      - 0
-      version: "0"
-required_rubygems_version: !ruby/object:Gem::Requirement 
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
   none: false
-  requirements: 
-  - - ">="
-    - !ruby/object:Gem::Version 
-      hash: 3
-      segments: 
-      - 0
-      version: "0"
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
 requirements: []
-
 rubyforge_project: 
 rubygems_version: 1.8.25
 signing_key: 
 specification_version: 3
-summary: Column based, timestamp replication of MySQL and Postgres databases. Uses Ruby for the glue code but pushes the heavy lifting on to the database.
-test_files: 
+summary: Column based, timestamp replication of MySQL and Postgres databases. Uses
+  Ruby for the glue code but pushes the heavy lifting on to the database.
+test_files:
 - spec/acceptance/loading_spec.rb
 - spec/acceptance_helper.rb
 - spec/database_helper.rb
@@ -193,4 +185,3 @@ test_files:
 - spec/unit/pipeline_spec.rb
 - spec/unit/stream_logger_spec.rb
 - spec/unit_helper.rb
-has_rdoc: false