mosql 0.3.2 → 0.4.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 43cb64bb52ef41955332728b063f24470d73c134
+  data.tar.gz: 9ddb444a959cdce77da58255fa64f4704cc4f201
+SHA512:
+  metadata.gz: 7260354422d7fad24eb8a018f7bf26108e7d3a86e502c0910df72cea011ecb58ee0c26065f8562940b4cca9bff2db06d07016aa4a83bd1dd4c1c0ee04c132f7d
+  data.tar.gz: 7bc1b499a41c0285770f26559c117e864e072490466f9aaabb7bb33093d5202767bc42d98525b2cf5a4405f79d88c5e4704df5aad75df918ad1b97ad1d5f15b8

data/.travis.yml

@@ -1,6 +1,7 @@
 language: ruby
 rvm:
   - 1.9.3
+  - 2.1.2
 services:
   - mongodb
   - postgresql

data/Gemfile.lock

@@ -1,12 +1,12 @@
 PATH
   remote: .
   specs:
-    mosql (0.3.2)
+    mosql (0.4.0)
       bson_ext
       json
       log4r
       mongo
-      mongoriver
+      mongoriver (= 0.4)
       pg
       rake
       sequel
@@ -14,24 +14,24 @@ PATH
 GEM
   remote: https://rubygems.org/
   specs:
-    bson (1.9.1)
-    bson_ext (1.9.1)
-      bson (~> 1.9.1)
+    bson (1.10.2)
+    bson_ext (1.10.2)
+      bson (~> 1.10.2)
     json (1.8.1)
     log4r (1.1.10)
     metaclass (0.0.4)
     minitest (3.0.0)
     mocha (1.0.0)
       metaclass (~> 0.0.1)
-    mongo (1.9.1)
-      bson (~> 1.9.1)
-    mongoriver (0.1.0)
+    mongo (1.10.2)
+      bson (= 1.10.2)
+    mongoriver (0.4.0)
       bson_ext
       log4r
       mongo (>= 1.7)
-    pg (0.14.1)
-    rake (10.3.1)
-    sequel (3.44.0)
+    pg (0.17.1)
+    rake (10.3.2)
+    sequel (4.14.0)
 
 PLATFORMS
   ruby

data/README.md

@@ -229,8 +229,16 @@ Patches and contributions are welcome! Please fork the project and
 open a pull request on [github][github], or just report issues.
 
 MoSQL includes a small but hopefully-growing test suite. It assumes a
-running PostgreSQL and MongoDB instance on the local host; You can
-point it at a different target via environment variables; See
-`test/functional/_lib.rb` for more information.
+running PostgreSQL and MongoDB instance on the local host. To run the
+test suite, first install all of MoSQL's dependencies:
+```shell
+bundle install
+```
+Then, run the tests:
+```shell
+rake test
+```
+You can also point the suite at a different target via environment
+variables; See `test/functional/_lib.rb` for more information.
 
 [github]: https://github.com/stripe/mosql

data/lib/mosql/schema.rb

@@ -70,6 +70,9 @@ module MoSQL
           end
         end
       end
+
+      # Lurky way to force Sequel force all timestamps to use UTC.
+      Sequel.default_timezone = :utc
     end
 
     def create_schema(db, clobber=false)
@@ -77,6 +80,8 @@ module MoSQL
         dbspec.each do |n, collection|
           next unless n.is_a?(String)
           meta = collection[:meta]
+          composite_key = meta[:composite_key]
+          keys = []
           log.info("Creating table '#{meta[:table]}'...")
           db.send(clobber ? :create_table! : :create_table?, meta[:table]) do
             collection[:columns].each do |col|
@@ -86,10 +91,14 @@ module MoSQL
               end
               column col[:name], col[:type], opts
 
-              if col[:source].to_sym == :_id
-                primary_key [col[:name].to_sym]
+              if composite_key and composite_key.include?(col[:name])
+                keys << col[:name].to_sym
+              elsif not composite_key and col[:source].to_sym == :_id
+                keys << col[:name].to_sym
               end
             end
+
+            primary_key keys
             if meta[:extra_props]
               type =
                 if meta[:extra_props] == "JSON"
@@ -159,6 +168,23 @@ module MoSQL
       end
     end
 
+    def transform_primitive(v, type=nil)
+      case v
+      when BSON::ObjectId, Symbol
+        v.to_s
+      when BSON::Binary
+        if type.downcase == 'uuid'
+          v.to_s.unpack("H*").first
+        else
+          Sequel::SQL::Blob.new(v.to_s)
+        end
+      when BSON::DBRef
+        v.object_id.to_s
+      else
+        v
+      end
+    end
+
     def transform(ns, obj, schema=nil)
       schema ||= find_ns!(ns)
 
@@ -174,18 +200,17 @@ module MoSQL
         else
           v = fetch_and_delete_dotted(obj, source)
           case v
-          when BSON::Binary, BSON::ObjectId, Symbol
-            v = v.to_s
-          when BSON::DBRef
-            v = v.object_id.to_s
           when Hash
-            v = JSON.dump(v)
+            v = JSON.dump(Hash[v.map { |k,v| [k, transform_primitive(v)] }])
           when Array
+            v = v.map { |it| transform_primitive(it) }
             if col[:array_type]
               v = Sequel.pg_array(v, col[:array_type])
             else
               v = JSON.dump(v)
             end
+          else
+            v = transform_primitive(v, type)
           end
         end
         row << v
@@ -268,6 +293,10 @@ module MoSQL
         'f'
       when Sequel::SQL::Function
         nil
+      when DateTime, Time
+        val.strftime("%FT%T.%6N %z")
+      when Sequel::SQL::Blob
+        "\\\\x" + [val].pack("h*")
       else
         val.to_s.gsub(/([\\\t\n\r])/, '\\\\\\1')
       end
@@ -290,7 +319,15 @@ module MoSQL
     end
 
     def primary_sql_key_for_ns(ns)
-      find_ns!(ns)[:columns].find {|c| c[:source] == '_id'}[:name]
+      ns = find_ns!(ns)
+      keys = []
+      if ns[:meta][:composite_key]
+        keys = ns[:meta][:composite_key]
+      else
+        keys << ns[:columns].find {|c| c[:source] == '_id'}[:name]
+      end
+
+      return keys
     end
   end
 end

data/lib/mosql/sql.rb

@@ -39,16 +39,24 @@ module MoSQL
       upsert!(table_for_ns(ns), @schema.primary_sql_key_for_ns(ns), h)
     end
 
-    # obj must contain an _id field. All other fields will be ignored.
     def delete_ns(ns, obj)
-      primary_sql_key = @schema.primary_sql_key_for_ns(ns)
+      primary_sql_keys = @schema.primary_sql_key_for_ns(ns)
       h = transform_one_ns(ns, obj)
-      raise "No #{primary_sql_key} found in transform of #{obj.inspect}" if h[primary_sql_key].nil?
-      table_for_ns(ns).where(primary_sql_key.to_sym => h[primary_sql_key]).delete
+      query = {}
+      primary_sql_keys.each do |key|
+        raise "No #{primary_sql_keys} found in transform of #{obj.inspect}" if h[key].nil?
+        query[key.to_sym] = h[key]
+      end
+
+      table_for_ns(ns).where(query).delete
     end
 
-    def upsert!(table, table_primary_key, item)
-      rows = table.where(table_primary_key.to_sym => item[table_primary_key]).update(item)
+    def upsert!(table, table_primary_keys, item)
+      query = {}
+      table_primary_keys.each do |key|
+        query[key.to_sym] = item[key]
+      end
+      rows = table.where(query).update(item)
       if rows == 0
         begin
           table.insert(item)
@@ -69,6 +77,10 @@ module MoSQL
       # how to get at this error code....
       e.wrapped_exception.result.error_field(PG::Result::PG_DIAG_SQLSTATE) == "23505"
     end
+
+    def self.duplicate_column_error?(e)
+      e.wrapped_exception.result.error_field(PG::Result::PG_DIAG_SQLSTATE) == "42701"
+    end
   end
 end
 

data/lib/mosql/streamer.rb

@@ -24,7 +24,7 @@ module MoSQL
     end
 
     def import
-      if options[:reimport] || tailer.read_timestamp.seconds == 0
+      if options[:reimport] || tailer.read_position.nil?
         initial_import
       end
     end
@@ -90,14 +90,17 @@ module MoSQL
       @schema.create_schema(@sql.db, !options[:no_drop_tables])
 
       unless options[:skip_tail]
-        start_ts = @mongo['local']['oplog.rs'].find_one({}, {:sort => [['$natural', -1]]})['ts']
+        start_state = {
+          'time' => nil,
+          'position' => @tailer.most_recent_position
+        }
       end
 
       dbnames = []
 
-      if dbname = options[:dbname]
-        log.info "Skipping DB scan and using db: #{dbname}"
-        dbnames = [ dbname ]
+      if options[:dbname]
+        log.info "Skipping DB scan and using db: #{options[:dbname]}"
+        dbnames = [ options[:dbname] ]
       else
         dbnames = @mongo.database_names
       end
@@ -121,8 +124,7 @@ module MoSQL
         end
       end
 
-
-      tailer.write_timestamp(start_ts) unless options[:skip_tail]
+      tailer.save_state(start_state) unless options[:skip_tail]
     end
 
     def did_truncate; @did_truncate ||= {}; end
@@ -164,9 +166,11 @@ module MoSQL
     end
 
     def optail
-      tailer.tail_from(options[:tail_from] ?
-                       BSON::Timestamp.new(options[:tail_from].to_i, 0) :
-                       nil)
+      tail_from = options[:tail_from]
+      if tail_from.is_a? Time
+        tail_from = tailer.most_recent_position(tail_from)
+      end
+      tailer.tail(:from => tail_from)
       until @done
         tailer.stream(1000) do |op|
           handle_op(op)
@@ -174,16 +178,21 @@ module MoSQL
       end
     end
 
-    def sync_object(ns, _id)
-      primary_sql_key = @schema.primary_sql_key_for_ns(ns)
-      sqlid           = @sql.transform_one_ns(ns, { '_id' => _id })[primary_sql_key]
-      obj             = collection_for_ns(ns).find_one({:_id => _id})
+    def sync_object(ns, selector)
+      obj = collection_for_ns(ns).find_one(selector)
       if obj
         unsafe_handle_exceptions(ns, obj) do
           @sql.upsert_ns(ns, obj)
         end
       else
-        @sql.table_for_ns(ns).where(primary_sql_key.to_sym => sqlid).delete()
+        primary_sql_keys = @schema.primary_sql_key_for_ns(ns)
+        schema = @schema.find_ns!(ns)
+        query = {}
+        primary_sql_keys.each do |key|
+          source =  schema[:columns].find {|c| c[:name] == key }[:source]
+          query[key] = selector[source]
+        end
+        @sql.table_for_ns(ns).where(query).delete()
       end
     end
 
@@ -194,6 +203,15 @@ module MoSQL
         return
       end
 
+      # First, check if this was an operation performed via applyOps. If so, call handle_op with
+      # for each op that was applied.
+      # The oplog format of applyOps commands can be viewed here:
+      # https://groups.google.com/forum/#!topic/mongodb-user/dTf5VEJJWvY
+      if op['op'] == 'c' && (ops = op['o']['applyOps'])
+        ops.each { |op| handle_op(op) }
+        return
+      end
+
       unless @schema.find_ns(op['ns'])
         log.debug("Skipping op for unknown ns #{op['ns']}...")
         return
@@ -218,15 +236,24 @@ module MoSQL
         update   = op['o']
         if update.keys.any? { |k| k.start_with? '$' }
           log.debug("resync #{ns}: #{selector['_id']} (update was: #{update.inspect})")
-          sync_object(ns, selector['_id'])
+          sync_object(ns, selector)
         else
-          log.debug("upsert #{ns}: _id=#{selector['_id']}")
 
           # The update operation replaces the existing object, but
           # preserves its _id field, so grab the _id off of the
           # 'query' field -- it's not guaranteed to be present on the
           # update.
-          update = { '_id' => selector['_id'] }.merge(update)
+          primary_sql_keys = @schema.primary_sql_key_for_ns(ns)
+          schema = @schema.find_ns!(ns)
+          keys = {}
+          primary_sql_keys.each do |key|
+            source =  schema[:columns].find {|c| c[:name] == key }[:source]
+            keys[key] = selector[source]
+          end
+
+          log.debug("upsert #{ns}: #{keys}")
+
+          update = keys.merge(update)
           unsafe_handle_exceptions(ns, update) do
             @sql.upsert_ns(ns, update)
           end

data/lib/mosql/tailer.rb

@@ -1,11 +1,25 @@
 module MoSQL
   class Tailer < Mongoriver::AbstractPersistentTailer
     def self.create_table(db, tablename)
-      db.create_table?(tablename) do
-        column :service,   'TEXT'
-        column :timestamp, 'INTEGER'
-        primary_key [:service]
+      if !db.table_exists?(tablename)
+        db.create_table(tablename) do
+          column :service,     'TEXT'
+          column :timestamp,   'INTEGER'
+          column :position,    'BYTEA'
+          primary_key [:service]
+        end
+      else
+        # Try to do seamless upgrades from before-tokumx times
+        # It will raise an exception in this in most cases,
+        # but there isn't a nice way I found to check if column
+        # exists.
+        begin
+          db.add_column(tablename, :position, 'BYTEA')
+        rescue Sequel::DatabaseError => e
+          raise unless MoSQL::SQLAdapter.duplicate_column_error?(e)
+        end
       end
+
       db[tablename.to_sym]
     end
 
@@ -15,25 +29,60 @@ module MoSQL
       @service = opts[:service] || "mosql"
     end
 
-    def read_timestamp
-      row = @table.where(:service => @service).select([:timestamp]).first
-      if row
-        BSON::Timestamp.new(row[:timestamp], 0)
+    def read_state
+      row = @table.where(:service => @service).first
+      return nil unless row
+      # Again, try to do seamless upgrades - 
+      # If latest operation before or at timestamp if no position 
+      # exists, use timestamp in database to guess what it could be.
+      result = {}
+      result['time'] = Time.at(row.fetch(:timestamp))
+      if row[:position]
+        result['position'] = from_blob(row[:position])
       else
-        BSON::Timestamp.new(0, 0)
+        log.warn("Trying to seamlessly update from old version!")
+        result['position'] = most_recent_position(result['time'])
+        save_state(result)
       end
+      result
     end
 
-    def write_timestamp(ts)
+    def write_state(state)
+      data = {
+        :service => @service,
+        :timestamp => state['time'].to_i,
+        :position => to_blob(state['position'])
+      }
+
       unless @did_insert
         begin
-          @table.insert({:service => @service, :timestamp => ts.seconds})
+          @table.insert(data)
         rescue Sequel::DatabaseError => e
           raise unless MoSQL::SQLAdapter.duplicate_key_error?(e)
         end
         @did_insert = true
       end
-      @table.where(:service => @service).update(:timestamp => ts.seconds)
+
+      @table.where(:service => @service).update(data)
+    end
+
+    private
+    def to_blob(position)
+      case database_type
+      when :mongo
+        return Sequel::SQL::Blob.new(position.seconds.to_s)
+      when :toku
+        return Sequel::SQL::Blob.new(position.to_s)
+      end
+    end
+
+    def from_blob(blob)
+      case database_type
+      when :mongo
+        return BSON::Timestamp.new(blob.to_i, 0)
+      when :toku
+        return BSON::Binary.new(blob)
+      end
     end
   end
 end

data/lib/mosql/version.rb

@@ -1,3 +1,3 @@
 module MoSQL
-  VERSION = "0.3.2"
+  VERSION = "0.4.0"
 end

data/mosql.gemspec

@@ -17,7 +17,8 @@ Gem::Specification.new do |gem|
   gem.version       = MoSQL::VERSION
 
   %w[sequel pg mongo bson_ext rake log4r json
-     mongoriver].each { |dep| gem.add_runtime_dependency(dep) }
+     ].each { |dep| gem.add_runtime_dependency(dep) }
+  gem.add_runtime_dependency "mongoriver", "0.4"
 
   gem.add_development_dependency "minitest"
   gem.add_development_dependency "mocha"

data/test/_lib.rb

@@ -1,3 +1,6 @@
+require 'rubygems'
+require 'bundler/setup'
+
 require 'minitest/autorun'
 require 'minitest/spec'
 require 'mocha'

data/test/functional/_lib.rb

@@ -58,6 +58,7 @@ EOF
     end
 
     def setup
+      Sequel.default_timezone = :utc
       @sequel = connect_sql
       @mongo  = connect_mongo
       super

data/test/functional/schema.rb

@@ -141,8 +141,10 @@ EOF
                             objects.map { |o| @specialmap.transform('db.collection', o) } )
       after = @sequel.select(Sequel.function(:NOW)).first[:now]
       rows = @sequel[:special].select.sort_by { |r| r[:_id] }
+
       assert_instance_of(Time, rows[0][:mosql_updated])
-      assert(rows[0][:mosql_updated] > before && rows[0][:mosql_updated] < after)
+      assert_operator(rows[0][:mosql_updated], :>, before)
+      assert_operator(rows[0][:mosql_updated], :<, after)
     end
   end
 end

data/test/functional/sql.rb

@@ -17,8 +17,8 @@ class MoSQL::Test::Functional::SQLTest < MoSQL::Test::Functional
 
   describe 'upsert' do
     it 'inserts new items' do
-      @adapter.upsert!(@table, '_id', {'_id' => 0, 'color' => 'red', 'quantity' => 10, 'numbers' => Sequel.pg_array([1, 2, 3], :integer)})
-      @adapter.upsert!(@table, '_id', {'_id' => 1, 'color' => 'blue', 'quantity' => 5, 'numbers' => Sequel.pg_array([], :integer)})
+      @adapter.upsert!(@table, ['_id'], {'_id' => 0, 'color' => 'red', 'quantity' => 10, 'numbers' => Sequel.pg_array([1, 2, 3], :integer)})
+      @adapter.upsert!(@table, ['_id'], {'_id' => 1, 'color' => 'blue', 'quantity' => 5, 'numbers' => Sequel.pg_array([], :integer)})
       assert_equal(2, @table.count)
       assert_equal('red',  @table[:_id => 0][:color])
       assert_equal(10,     @table[:_id => 0][:quantity])
@@ -29,11 +29,11 @@ class MoSQL::Test::Functional::SQLTest < MoSQL::Test::Functional
     end
 
     it 'updates items' do
-      @adapter.upsert!(@table, '_id', {'_id' => 0, 'color' => 'red', 'quantity' => 10})
+      @adapter.upsert!(@table, ['_id'], {'_id' => 0, 'color' => 'red', 'quantity' => 10})
       assert_equal(1, @table.count)
       assert_equal('red',  @table[:_id => 0][:color])
 
-      @adapter.upsert!(@table, '_id', {'_id' => 0, 'color' => 'blue', 'quantity' => 5})
+      @adapter.upsert!(@table, ['_id'], {'_id' => 0, 'color' => 'blue', 'quantity' => 5})
       assert_equal(1, @table.count)
       assert_equal('blue', @table[:_id => 0][:color])
     end

data/test/functional/streamer.rb

@@ -53,6 +53,22 @@ filter_test:
     :columns:
       - _id: TEXT
       - var: INTEGER
+
+composite_key_test:
+  collection:
+    :meta:
+      :table: composite_table
+      :composite_key:
+        - store
+        - time
+    :columns:
+      - store:
+        :source: _id.s
+        :type: TEXT
+      - time:
+        :source: _id.t
+        :type: TIMESTAMP
+      - var: TEXT
 EOF
 
     before do
@@ -61,6 +77,7 @@ EOF
 
       @sequel.drop_table?(:sqltable)
       @sequel.drop_table?(:sqltable2)
+      @sequel.drop_table?(:composite_table)
       @map.create_schema(@sequel)
 
       @streamer = build_streamer
@@ -78,6 +95,22 @@ EOF
       assert_equal(27, sequel[:sqltable].where(:_id => o['_id'].to_s).select.first[:var])
     end
 
+    it 'applies ops performed via applyOps' do
+      o = { '_id' => BSON::ObjectId.new, 'var' => 17 }
+      @adapter.upsert_ns('mosql_test.collection', o)
+
+      op = { 'ns' => 'mosql_test.collection',
+             'op' => 'u',
+             'o2' => { '_id' => o['_id'] },
+             'o'  => { 'var' => 27 }
+           }
+      @streamer.handle_op({ 'op' => 'c',
+                            'ns' => 'mosql_test.$cmd',
+                            'o' => { 'applyOps' => [op] }
+                          })
+      assert_equal(27, sequel[:sqltable].where(:_id => o['_id'].to_s).select.first[:var])
+    end
+
     it 'handle "d" ops with BSON::ObjectIds' do
       o = { '_id' => BSON::ObjectId.new, 'var' => 17 }
       @adapter.upsert_ns('mosql_test.collection', o)
@@ -151,6 +184,39 @@ EOF
       assert_equal(data[1], record)
     end
 
+    it 'handles "u" ops with a compsite key' do
+      date = Time.utc(2014, 7, 1)
+      o = {'_id' => {'s' => 'asdf', 't' => date}, 'var' => 'data'}
+      collection = mongo["composite_key_test"]["collection"]
+      collection.drop
+      collection.insert(o)
+
+      @streamer.options[:skip_tail] = true
+      @streamer.initial_import
+
+      collection.update({ '_id' => { 's' => 'asdf', 't' => date}}, { '$set' => { 'var' => 'new_data'}})
+      @streamer.handle_op({'ns' => 'composite_key_test.collection',
+                           'op' => 'u',
+                           'o2' => { '_id' => { 's' => 'asdf', 't' => date}},
+                           'o'  => { '$set' => { 'var' => 'new_data'}}
+                           })
+
+      assert_equal(0, @sequel[:composite_table].where(:var => "data").count)
+      assert_equal(1, @sequel[:composite_table].where(:var => "new_data").count)
+    end
+
+    it 'handles composite keys' do
+      o = {'_id' => {'s' => 'asdf', 't' => Time.new}, 'var' => 'data'}
+      collection = mongo["composite_key_test"]["collection"]
+      collection.drop
+      collection.insert(o)
+
+      @streamer.options[:skip_tail] = true
+      @streamer.initial_import
+
+      assert_equal(1, @sequel[:composite_table].count)
+    end
+
     describe '.bulk_upsert' do
       it 'inserts multiple rows' do
         objs = [
@@ -252,4 +318,56 @@ EOF
       assert_equal(ids.map(&:to_s).sort, sqlobjs.map { |o| o[:_id] }.sort)
     end
   end
+  describe 'timestamps' do
+  TIMESTAMP_MAP = <<EOF
+---
+db:
+  has_timestamp:
+    :meta:
+      :table: has_timestamp
+    :columns:
+      - _id: TEXT
+      - ts: timestamp
+EOF
+
+    before do
+      @map = MoSQL::Schema.new(YAML.load(TIMESTAMP_MAP))
+      @adapter = MoSQL::SQLAdapter.new(@map, sql_test_uri)
+
+      mongo['db']['has_timestamp'].drop
+      @sequel.drop_table?(:has_timestamp)
+      @map.create_schema(@sequel)
+
+      @streamer = build_streamer
+    end
+
+    it 'preserves milliseconds on import' do
+      ts = Time.utc(2014, 8, 7, 6, 54, 32, 123000)
+      mongo['db']['has_timestamp'].insert({ts: ts})
+      @streamer.options[:skip_tail] = true
+      @streamer.initial_import
+
+      row = @sequel[:has_timestamp].select.to_a
+      assert_equal(1, row.length)
+      assert_equal(ts.to_i, row.first[:ts].to_i)
+      assert_equal(ts.tv_usec, row.first[:ts].tv_usec)
+    end
+
+    it 'preserves milliseconds on tailing' do
+      ts = Time.utc(2006,01,02, 15,04,05,678000)
+      id = mongo['db']['has_timestamp'].insert({ts: ts})
+      @streamer.handle_op(
+        {
+          "ts" => {"t" => 1408647630, "i" => 4},
+          "h"  => -965650193548512059,
+          "v"  => 2,
+          "op" => "i",
+          "ns" => "db.has_timestamp",
+          "o"  => mongo['db']['has_timestamp'].find_one({_id: id})
+        })
+      got = @sequel[:has_timestamp].where(:_id => id.to_s).select.first[:ts]
+      assert_equal(ts.to_i, got.to_i)
+      assert_equal(ts.tv_usec, got.tv_usec)
+    end
+  end
 end

data/test/functional/transform.rb

@@ -0,0 +1,122 @@
+require File.join(File.dirname(__FILE__), '_lib.rb')
+require 'mosql/cli'
+
+class MoSQL::Test::Functional::TransformTest < MoSQL::Test::Functional
+  TESTCASES = [
+    [
+      BSON::ObjectId.from_string('5405fae77c584947fc000001'),
+      'TEXT',
+      '5405fae77c584947fc000001'
+    ],
+    [
+      Time.utc(2006,01,02, 15,04,05,678000),
+      'TIMESTAMP',
+      Time.utc(2006,01,02, 15,04,05,678000)
+    ],
+    [
+      :stringy,
+      'TEXT',
+      'stringy'
+    ],
+    [
+      BSON::DBRef.new('db.otherns', BSON::ObjectId.from_string('5405fae77c584947fc000001')),
+      'TEXT',
+      '5405fae77c584947fc000001'
+    ],
+    [
+      [
+        BSON::ObjectId.from_string('5405fae77c584947fc000001'),
+        BSON::ObjectId.from_string('5405fae77c584947fc000002')
+      ],
+      'TEXT ARRAY',
+      ['5405fae77c584947fc000001', '5405fae77c584947fc000002']
+    ],
+    [
+      [
+        BSON::ObjectId.from_string('5405fae77c584947fc000001'),
+        BSON::ObjectId.from_string('5405fae77c584947fc000002')
+      ],
+      'TEXT',
+      ['5405fae77c584947fc000001', '5405fae77c584947fc000002'].to_json,
+    ],
+    [
+      [
+        BSON::DBRef.new('db.otherns', BSON::ObjectId.from_string('5405fae77c584947fc000001')),
+        BSON::DBRef.new('db.otherns', BSON::ObjectId.from_string('5405fae77c584947fc000002'))
+      ],
+      'TEXT ARRAY',
+      ['5405fae77c584947fc000001', '5405fae77c584947fc000002']
+    ],
+    [
+      [
+        BSON::DBRef.new('db.otherns', BSON::ObjectId.from_string('5405fae77c584947fc000001')),
+        BSON::DBRef.new('db.otherns', BSON::ObjectId.from_string('5405fae77c584947fc000002'))
+      ],
+      'TEXT',
+      ['5405fae77c584947fc000001', '5405fae77c584947fc000002'].to_json
+    ],
+    [
+      BSON::Binary.new(["2d931510d99f494a8c6787feb05e1594"].pack("H*"),
+        BSON::Binary::SUBTYPE_UUID),
+      'UUID',
+      "2d931510-d99f-494a-8c67-87feb05e1594"
+    ],
+    [
+      BSON::Binary.new(["deadbeefcafebabe"].pack("H*"),
+        BSON::Binary::SUBTYPE_SIMPLE),
+      'BYTEA',
+      ["deadbeefcafebabe"].pack("H*")
+    ]
+  ]
+
+  TESTCASES.each do |mongo, typ, sql|
+    it "Can transform a #{mongo.class} into a #{typ}" do
+      map = {'test' => {'test_transform' =>
+          {
+            meta: {
+              table: 'test_transform'
+            },
+            columns: [
+              {'_id'   => 'TEXT'},
+              {'value' => typ},
+            ]
+          }}}
+      schema = MoSQL::Schema.new(map)
+      adapter = MoSQL::SQLAdapter.new(schema, sql_test_uri)
+      @sequel.drop_table?(:test_transform)
+      collection = @mongo['test']['test_transform']
+      collection.drop
+
+      schema.create_schema(@sequel)
+      streamer = MoSQL::Streamer.new(:mongo => self.mongo,
+        :tailer => nil,
+        :options => {skip_tail: true},
+        :sql => adapter,
+        :schema => schema)
+
+      # Test initial import
+      id = 'imported'
+      collection.insert({_id: id, value: mongo})
+      streamer.initial_import
+
+      got = @sequel[:test_transform].where(_id: id).to_a
+      assert_equal(sql, got.first[:value], "was able to transform a #{typ} field on initial import")
+
+      # Test streaming an insert
+      id = 'inserted'
+      collection.insert({_id: id, value: mongo})
+      streamer.handle_op(
+        {
+          "ts" => {"t" => 1408647630, "i" => 4},
+          "h"  => -965650193548512059,
+          "v"  => 2,
+          "op" => "i",
+          "ns" => "test.test_transform",
+          "o"  => collection.find_one(_id: id)
+        })
+
+      got = @sequel[:test_transform].where(_id: id).to_a
+      assert_equal(sql, got.first[:value], "was able to transform a #{typ} field while streaming")
+    end
+  end
+end

data/test/unit/lib/mosql/schema.rb

@@ -39,6 +39,23 @@ db:
       - arry: TEXT
     :meta:
       :table: sqltable5
+  with_composite_key:
+    :meta:
+      :table: sqltable6
+      :composite_key:
+        - store
+        - time
+    :columns:
+      - store:
+        :source: _id.s
+        :type: TEXT
+      - time:
+        :source: id.t
+        :type: TEXT
+      - var:
+        :source: var
+        :type: TEXT
+
 EOF
 
   before do
@@ -87,8 +104,8 @@ EOF
   end
 
   it 'Can find the primary key of the SQL table' do
-    assert_equal('id', @map.primary_sql_key_for_ns('db.collection'))
-    assert_equal('_id', @map.primary_sql_key_for_ns('db.old_conf_syntax'))
+    assert_equal(['id'], @map.primary_sql_key_for_ns('db.collection'))
+    assert_equal(['_id'], @map.primary_sql_key_for_ns('db.old_conf_syntax'))
   end
 
   it 'can create a SQL schema' do
@@ -98,6 +115,7 @@ EOF
     db.expects(:create_table?).with('sqltable3')
     db.expects(:create_table?).with('sqltable4')
     db.expects(:create_table?).with('sqltable5')
+    db.expects(:create_table?).with('sqltable6')
 
     @map.create_schema(db)
   end
@@ -127,6 +145,11 @@ EOF
     stub_5.expects(:column).with('_id', 'TEXT', {})
     stub_5.expects(:column).with('arry', 'TEXT', {})
     stub_5.expects(:primary_key).with([:_id])
+    stub_6 = stub('table 6')
+    stub_6.expects(:column).with('store', 'TEXT', {})
+    stub_6.expects(:column).with('time', 'TEXT', {})
+    stub_6.expects(:column).with('var', 'TEXT', {})
+    stub_6.expects(:primary_key).with([:store, :time])
     (class << db; self; end).send(:define_method, :create_table?) do |tbl, &blk|
       case tbl
       when "sqltable"
@@ -139,6 +162,8 @@ EOF
         o = stub_4
       when "sqltable5"
         o = stub_5
+      when "sqltable6"
+        o = stub_6
       else
         assert(false, "Tried to create an unexpected table: #{tbl}")
       end
@@ -177,6 +202,13 @@ EOF
       assert_equal(["row 1", nil, oid.to_s, nil], out)
     end
 
+    it 'converts DBRef to object id in arrays' do
+      oid = [ BSON::ObjectId.new, BSON::ObjectId.new]
+      o = {'_id' => "row 1", "str" => [ BSON::DBRef.new('db.otherns', oid[0]), BSON::DBRef.new('db.otherns', oid[1]) ] }
+      out = @map.transform('db.collection', o)
+      assert_equal(["row 1", nil, JSON.dump(oid.map! {|o| o.to_s}), nil ], out)
+    end
+
     it 'changes NaN to null in extra_props' do
       out = @map.transform('db.with_extra_props', {'_id' => 7, 'nancy' => 0.0/0.0})
       extra = JSON.parse(out[1])

metadata

@@ -1,174 +1,153 @@
 --- !ruby/object:Gem::Specification
 name: mosql
 version: !ruby/object:Gem::Version
-  version: 0.3.2
-  prerelease: 
+  version: 0.4.0
 platform: ruby
 authors:
 - Nelson Elhage
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-08-11 00:00:00.000000000 Z
+date: 2014-10-01 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: sequel
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
   name: pg
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
   name: mongo
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
   name: bson_ext
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
   name: log4r
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
   name: json
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
   name: mongoriver
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - '='
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '0.4'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - '='
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '0.4'
 - !ruby/object:Gem::Dependency
   name: minitest
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
   name: mocha
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
 description: A library for streaming MongoDB to SQL
@@ -179,8 +158,8 @@ executables:
 extensions: []
 extra_rdoc_files: []
 files:
-- .gitignore
-- .travis.yml
+- ".gitignore"
+- ".travis.yml"
 - Gemfile
 - Gemfile.lock
 - LICENSE
@@ -198,41 +177,40 @@ files:
 - mosql.gemspec
 - test/_lib.rb
 - test/functional/_lib.rb
-- test/functional/functional.rb
 - test/functional/schema.rb
 - test/functional/sql.rb
 - test/functional/streamer.rb
+- test/functional/transform.rb
 - test/unit/lib/mosql/schema.rb
 homepage: https://github.com/stripe/mosql
 licenses: []
+metadata: {}
 post_install_message: 
 rdoc_options: []
 require_paths:
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement
-  none: false
   requirements:
-  - - ! '>='
+  - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
-  none: false
   requirements:
-  - - ! '>='
+  - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 1.8.23.2
+rubygems_version: 2.2.2
 signing_key: 
-specification_version: 3
+specification_version: 4
 summary: MongoDB -> SQL streaming bridge
 test_files:
 - test/_lib.rb
 - test/functional/_lib.rb
-- test/functional/functional.rb
 - test/functional/schema.rb
 - test/functional/sql.rb
 - test/functional/streamer.rb
+- test/functional/transform.rb
 - test/unit/lib/mosql/schema.rb
 has_rdoc: 

data/test/functional/functional.rb

@@ -1,7 +0,0 @@
-require File.join(File.dirname(__FILE__), '_lib.rb')
-
-class MoSQL::Test::Functional::SchemaTest < MoSQL::Test::Functional
-  it 'runs tests' do
-    assert(true)
-  end
-end