sq-dbsync 1.0.6 → 1.0.7

data/README.md

@@ -171,6 +171,20 @@ Documentation
 * `timestamp` The column to treat as a timestamp. Must be a member of the
   `:columns` option. (default: select `updated_at` or `created_at`, in that
   order)
+* `type_casts` a hash for specifying column-level type casting from postgres
+  data types to mysql data types. Primarily intended for custom postgres type
+  defintions that have no mysql equivalent. The hash key is the column name,
+  and the value is the desired mysql column type. See the
+  `Sq::Dbsync::Database::Postgres::CASTS` constant for the format of the hash.
+  Example:
+
+```ruby
+{
+  :table_name => :some_table,
+  :columns => [ :custom_postgres_type ],
+  :type_casts => { :custom_postgres_type => "varchar(255)" }
+}
+```
 
 ### Handling Deletes
 

data/lib/sq/dbsync/batch_load_action.rb

@@ -53,8 +53,7 @@ module Sq::Dbsync
 
     def filter_columns
       source = plan.source_db
-      source_columns = source.hash_schema(plan.source_table_name).keys
-
+      source_columns = source.hash_schema(plan).keys
       plan.columns = resolve_columns(plan, source_columns)
     end
 

data/lib/sq/dbsync/database/common.rb

@@ -32,9 +32,9 @@ module Sq::Dbsync::Database
       sql
     end
 
-    def hash_schema(table_name)
+    def hash_schema(plan)
       ensure_connection
-      Hash[schema(table_name)]
+      Hash[schema(plan.source_table_name)]
     end
 
     def name
@@ -94,6 +94,5 @@ module Sq::Dbsync::Database
         Process.waitpid2(pid)[1]
       end
     end
-
   end
 end

data/lib/sq/dbsync/database/postgres.rb

@@ -8,6 +8,24 @@ module Sq::Dbsync::Database
   # Decorator around a Sequel database object, providing some non-standard
   # extensions required for effective extraction from Postgres.
   class Postgres < Delegator
+    CASTS = {
+        "text" => "varchar(255)",
+        "character varying(255)" => "varchar(255)",
+
+        # 255 is an arbitrary choice here. The one example we have
+        # only has data 32 characters long in it.
+        "character varying"      => "varchar(255)",
+
+        # Arbitrarily chosen precision. The default numeric type in mysql is
+        # (10, 0), which is perhaps the most useless default I could imagine.
+        "numeric" => "numeric(12,6)",
+        "boolean" => "char(1)",
+
+        # mysql has no single-column representation for timestamp with time zone
+        "timestamp with time zone" => "datetime",
+        "time without time zone" => "time",
+        "timestamp without time zone" => "datetime",
+      }
 
     include Sq::Dbsync::Database::Common
 
@@ -22,12 +40,15 @@ module Sq::Dbsync::Database
       # Unimplemented
     end
 
-    def hash_schema(table_name)
+    def hash_schema(plan)
+      type_casts = plan.type_casts || {}
       ensure_connection
 
-      result = schema(table_name).each do |col, metadata|
+      result = schema(plan.source_table_name).each do |col, metadata|
         metadata[:source_db_type] ||= metadata[:db_type]
-        metadata[:db_type] = psql_to_mysql_conversion(metadata[:db_type])
+        metadata[:db_type] = cast_psql_to_mysql(
+          metadata[:db_type], type_casts[col.to_s]
+        )
       end
 
       Hash[result]
@@ -37,35 +58,14 @@ module Sq::Dbsync::Database
 
     attr_reader :db
 
-    def psql_to_mysql_conversion(db_type)
-      {
-        "text" => "varchar(255)",
-        "character varying(255)" => "varchar(255)",
-
-        # 255 is an arbitrary choice here. The one example we have
-        # only has data 32 characters long in it.
-        "character varying"      => "varchar(255)",
-
-        # Arbitrarily chosen precision. The default numeric type in mysql is
-        # (10, 0), which is perhaps the most useless default I could imagine.
-        "numeric" => "numeric(12,6)",
-
-        "time without time zone" => "time",
-        "timestamp without time zone" => "datetime",
-
-        # mysql has no single-column representation for timestamp with time zone
-        "timestamp with time zone" => "datetime",
-
-        "boolean" => "char(1)"
-      }.fetch(db_type, db_type)
+    def cast_psql_to_mysql(db_type, cast=nil)
+      CASTS.fetch(db_type, cast || db_type)
     end
 
-
     # 'with time zone' pg fields have to be selected as
     # "column-name"::timestamp or they end up with a +XX tz offset on
     # them, which isn't the correct format for timestamp fields (which
     # is what they get turned into for portability.)
-
     def customize_sql(sql, schema)
       schema.each do |name, metadata|
         if metadata[:source_db_type].end_with? "with time zone"

data/lib/sq/dbsync/incremental_load_action.rb

@@ -72,7 +72,7 @@ module Sq::Dbsync
 
     def filter_columns
       source         = plan.source_db
-      source_columns = source.hash_schema(plan.source_table_name).keys
+      source_columns = source.hash_schema(plan).keys
       plan.columns   = resolve_columns(plan, source_columns) &
         (target_columns || source_columns)
     end
@@ -81,8 +81,7 @@ module Sq::Dbsync
       # Because we may create the target table later if necessary,
       # we need to check if it *really* exists
       target_columns = if target.table_exists?(plan.table_name)
-        tname = "#{prefix}#{plan.table_name}"
-        target.hash_schema(tname).keys
+        target.hash_schema(plan).keys
       else
         nil
       end

data/lib/sq/dbsync/load_action.rb

@@ -83,9 +83,9 @@ module Sq::Dbsync
       end
     end
 
-    def add_schema_to_table_plan(x)
-      x.schema ||= x.source_db.hash_schema(x.source_table_name)
-      x
+    def add_schema_to_table_plan(plan)
+      plan.schema ||= plan.source_db.hash_schema(plan)
+      plan
     end
 
     def resolve_columns(plan, source_columns)

data/lib/sq/dbsync/refresh_recent_load_action.rb

@@ -49,7 +49,7 @@ module Sq::Dbsync
 
     def filter_columns
       source         = plan.source_db
-      source_columns = source.hash_schema(plan.source_table_name).keys
+      source_columns = source.hash_schema(plan).keys
       plan.columns   = resolve_columns(plan, source_columns) &
         (target_columns || source_columns)
     end
@@ -58,7 +58,7 @@ module Sq::Dbsync
       # Because we may create the target table later if necessary,
       # we need to check if it *really* exists
       target_columns = if target.table_exists?(plan.table_name)
-        target.hash_schema(plan.table_name).keys
+        target.hash_schema(plan).keys
       else
         nil
       end

data/lib/sq/dbsync/version.rb

@@ -1,5 +1,5 @@
 module Sq
   module Dbsync
-    VERSION = '1.0.6'
+    VERSION = '1.0.7'
   end
 end

data/spec/integration/batch_load_action_spec.rb

@@ -1,5 +1,4 @@
 require 'integration_helper'
-
 require 'sq/dbsync/database/connection'
 require 'sq/dbsync/loggers'
 require 'sq/dbsync/batch_load_action'
@@ -54,8 +53,9 @@ describe SQD::BatchLoadAction do
       it 'copies all columns to target' do
         action.call
 
-        target.hash_schema(:test_table).keys.should ==
-          source.hash_schema(:test_table).keys
+        plan = OpenStruct.new(table_plan)
+        target.hash_schema(plan).keys.should ==
+          source.hash_schema(plan).keys
       end
     end
 

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: sq-dbsync
 version: !ruby/object:Gem::Version
-  version: 1.0.6
+  version: 1.0.7
   prerelease: 
 platform: ruby
 authors:
@@ -10,7 +10,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-05-21 00:00:00.000000000 Z
+date: 2013-05-29 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rspec